//===--- IRGenPIL.cpp - Swift Per-Function IR Generation ------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
//  This file implements basic setup and teardown for the class which
//  performs IR generation for function bodies.
//
//===----------------------------------------------------------------------===//

#define DEBUG_TYPE "irgenpil"

#include "llvm/IR/DIBuilder.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/TinyPtrVector.h"
#include "llvm/Support/SaveAndRestore.h"
#include "llvm/Support/Debug.h"
#include "llvm/Transforms/Utils/Local.h"
#include "clang/AST/ASTContext.h"
#include "clang/Basic/TargetInfo.h"

#include "polarphp/basic/ExternalUnion.h"
#include "polarphp/basic/Range.h"
#include "polarphp/basic/StlExtras.h"
#include "polarphp/ast/AstContext.h"
#include "polarphp/ast/IRGenOptions.h"
#include "polarphp/ast/Pattern.h"
#include "polarphp/ast/ParameterList.h"
#include "polarphp/ast/SubstitutionMap.h"
#include "polarphp/ast/Types.h"
#include "polarphp/pil/lang/ApplySite.h"
#include "polarphp/pil/lang/Dominance.h"
#include "polarphp/pil/lang/PrettyStackTrace.h"
#include "polarphp/pil/lang/PILDebugScope.h"
#include "polarphp/pil/lang/PILDeclRef.h"
#include "polarphp/pil/lang/PILLinkage.h"
#include "polarphp/pil/lang/PILModule.h"
#include "polarphp/pil/lang/PILType.h"
#include "polarphp/pil/lang/PILVisitor.h"
#include "polarphp/pil/lang/InstructionUtils.h"
#include "clang/CodeGen/CodeGenABITypes.h"

#include "polarphp/irgen/internal/CallEmission.h"
#include "polarphp/irgen/internal/Explosion.h"
#include "polarphp/irgen/internal/GenArchetype.h"
#include "polarphp/irgen/internal/GenBuiltin.h"
#include "polarphp/irgen/internal/GenCall.h"
#include "polarphp/irgen/internal/GenCast.h"
#include "polarphp/irgen/internal/GenClass.h"
#include "polarphp/irgen/internal/GenConstant.h"
#include "polarphp/irgen/internal/GenDecl.h"
#include "polarphp/irgen/internal/GenEnum.h"
#include "polarphp/irgen/internal/GenExistential.h"
#include "polarphp/irgen/internal/GenFunc.h"
#include "polarphp/irgen/internal/GenHeap.h"
#include "polarphp/irgen/internal/GenIntegerLiteral.h"
#include "polarphp/irgen/internal/GenOpaque.h"
#include "polarphp/irgen/internal/GenPoly.h"
#include "polarphp/irgen/internal/GenInterface.h"
#include "polarphp/irgen/internal/GenStruct.h"
#include "polarphp/irgen/internal/GenTuple.h"
#include "polarphp/irgen/internal/GenType.h"
#include "polarphp/irgen/internal/IRGenDebugInfo.h"
#include "polarphp/irgen/internal/IRGenModule.h"
#include "polarphp/irgen/internal/MetadataLayout.h"
#include "polarphp/irgen/internal/MetadataRequest.h"
#include "polarphp/irgen/internal/NativeConventionSchema.h"
#include "polarphp/irgen/internal/ReferenceTypeInfo.h"

using namespace polar;
using namespace irgen;

namespace {

class LoweredValue;

struct DynamicallyEnforcedAddress {
   Address Addr;
   llvm::Value *ScratchBuffer;
};

struct CoroutineState {
   Address Buffer;
   llvm::Value *Continuation;
   TemporarySet Temporaries;
};

/// Represents a PIL value lowered to IR, in one of these forms:
/// - an Address, corresponding to a PIL address value;
/// - an Explosion of (unmanaged) Values, corresponding to a PIL "register"; or
/// - a CallEmission for a partially-applied curried function or method.
class LoweredValue {
public:
   enum class Kind {
      /// The first two LoweredValue kinds correspond to a PIL address value.
      ///
      /// The LoweredValue of an existential alloc_stack keeps an owning container
      /// in addition to the address of the allocated buffer.
      /// Depending on the allocated type, the container may be equal to the
      /// buffer itself (for types with known sizes) or it may be the address
      /// of a fixed-size container which points to the heap-allocated buffer.
      /// In this case the address-part may be null, which means that the buffer
      /// is not allocated yet.
         ContainedAddress,

      /// The LoweredValue of a resilient, generic, or loadable typed alloc_stack
      /// keeps an optional stackrestore point in addition to the address of the
      /// allocated buffer. For all other address values the stackrestore point is
      /// just null.
      /// If the stackrestore point is set (currently, this might happen for
      /// opaque types: generic and resilient) the deallocation of the stack must
      /// reset the stack pointer to this point.
         StackAddress,

      /// A @box together with the address of the box value.
         OwnedAddress,

      /// The lowered value of a begin_access instruction using dynamic
      /// enforcement.
         DynamicallyEnforcedAddress,

      /// A normal value, represented as an exploded array of llvm Values.
         ExplosionVector,

      /// The special case of a single explosion.
         SingletonExplosion,

      /// A value that represents a function pointer.
         FunctionPointer,

      /// A value that represents an Objective-C method that must be called with
      /// a form of objc_msgSend.
         ObjCMethod,

      /// The special case of an empty explosion.
         EmptyExplosion,

      /// A coroutine state.
         CoroutineState,
   };

   Kind kind;

private:
   using ExplosionVector = SmallVector<llvm::Value *, 4>;
   using SingletonExplosion = llvm::Value*;

   using Members = ExternalUnionMembers<ContainedAddress,
      StackAddress,
      OwnedAddress,
      DynamicallyEnforcedAddress,
      ExplosionVector,
      SingletonExplosion,
      FunctionPointer,
//      ObjCMethod,
      CoroutineState,
      void>;

   static Members::Index getMemberIndexForKind(Kind kind) {
      switch (kind) {
         case Kind::ContainedAddress: return Members::indexOf<ContainedAddress>();
         case Kind::StackAddress: return Members::indexOf<StackAddress>();
         case Kind::OwnedAddress: return Members::indexOf<OwnedAddress>();
         case Kind::DynamicallyEnforcedAddress: return Members::indexOf<DynamicallyEnforcedAddress>();
         case Kind::ExplosionVector: return Members::indexOf<ExplosionVector>();
         case Kind::SingletonExplosion: return Members::indexOf<SingletonExplosion>();
         case Kind::FunctionPointer: return Members::indexOf<FunctionPointer>();
//         case Kind::ObjCMethod: return Members::indexOf<ObjCMethod>();
         case Kind::CoroutineState: return Members::indexOf<CoroutineState>();
         case Kind::EmptyExplosion: return Members::indexOf<void>();
      }
      llvm_unreachable("bad kind");
   }
   ExternalUnion<Kind, Members, getMemberIndexForKind> Storage;

public:

   /// Create an address value without a stack restore point.
   LoweredValue(const Address &address)
      : kind(Kind::StackAddress) {
      Storage.emplace<StackAddress>(kind, address);
   }

   /// Create an address value with an optional stack restore point.
   LoweredValue(const StackAddress &address)
      : kind(Kind::StackAddress) {
      Storage.emplace<StackAddress>(kind, address);
   }

   /// Create an address value using dynamic enforcement.
   LoweredValue(const DynamicallyEnforcedAddress &address)
      : kind(Kind::DynamicallyEnforcedAddress) {
      Storage.emplace<DynamicallyEnforcedAddress>(kind, address);
   }

   enum ContainerForUnallocatedAddress_t { ContainerForUnallocatedAddress };

   /// Create an address value for an alloc_stack, consisting of a container and
   /// a not yet allocated buffer.
   LoweredValue(const Address &container, ContainerForUnallocatedAddress_t)
      : kind(Kind::ContainedAddress) {
      Storage.emplace<ContainedAddress>(kind, container, Address());
   }

   /// Create an address value for an alloc_stack, consisting of a container and
   /// the address of the allocated buffer.
   LoweredValue(const ContainedAddress &address)
      : kind(Kind::ContainedAddress) {
      Storage.emplace<ContainedAddress>(kind, address);
   }

   LoweredValue(const FunctionPointer &fn)
      : kind(Kind::FunctionPointer) {
      Storage.emplace<FunctionPointer>(kind, fn);
   }
   // @todo

//   LoweredValue(ObjCMethod &&objcMethod)
//      : kind(Kind::ObjCMethod) {
//      Storage.emplace<ObjCMethod>(kind, std::move(objcMethod));
//   }

   LoweredValue(Explosion &e) {
      auto elts = e.claimAll();
      if (elts.empty()) {
         kind = Kind::EmptyExplosion;
      } else if (elts.size() == 1) {
         kind = Kind::SingletonExplosion;
         Storage.emplace<SingletonExplosion>(kind, elts.front());
      } else {
         kind = Kind::ExplosionVector;
         auto &explosion = Storage.emplace<ExplosionVector>(kind);
         explosion.append(elts.begin(), elts.end());
      }
   }

   LoweredValue(const OwnedAddress &boxWithAddress)
      : kind(Kind::OwnedAddress) {
      Storage.emplace<OwnedAddress>(kind, boxWithAddress);
   }

   LoweredValue(CoroutineState &&state)
      : kind(Kind::CoroutineState) {
      Storage.emplace<CoroutineState>(kind, std::move(state));
   }

   LoweredValue(LoweredValue &&lv)
      : kind(lv.kind) {
      Storage.moveConstruct(kind, std::move(lv.Storage));
   }

   LoweredValue &operator=(LoweredValue &&lv) {
      Storage.moveAssign(kind, lv.kind, std::move(lv.Storage));
      kind = lv.kind;
      return *this;
   }

   ~LoweredValue() {
      Storage.destruct(kind);
   }

   bool isAddress() const {
      return (kind == Kind::StackAddress ||
              kind == Kind::DynamicallyEnforcedAddress);
   }
   bool isUnallocatedAddressInBuffer() const {
      return kind == Kind::ContainedAddress &&
             !Storage.get<ContainedAddress>(kind).getAddress().isValid();
   }
   bool isBoxWithAddress() const {
      return kind == Kind::OwnedAddress;
   }

   const StackAddress &getStackAddress() const {
      return Storage.get<StackAddress>(kind);
   }

   Address getContainerOfAddress() const {
      const auto &containedAddress = Storage.get<ContainedAddress>(kind);
      assert(containedAddress.getContainer().isValid() && "address has no container");
      return containedAddress.getContainer();
   }

   Address getAddressInContainer() const {
      const auto &containedAddress = Storage.get<ContainedAddress>(kind);
      assert(containedAddress.getContainer().isValid() &&
             "address has no container");
      return containedAddress.getAddress();
   }

   const DynamicallyEnforcedAddress &getDynamicallyEnforcedAddress() const {
      return Storage.get<DynamicallyEnforcedAddress>(kind);
   }

   Address getAnyAddress() const {
      if (kind == LoweredValue::Kind::StackAddress) {
         return Storage.get<StackAddress>(kind).getAddress();
      } else if (kind == LoweredValue::Kind::ContainedAddress) {
         return getAddressInContainer();
      } else {
         return getDynamicallyEnforcedAddress().Addr;
      }
   }

   Address getAddressOfBox() const {
      return Storage.get<OwnedAddress>(kind).getAddress();
   }

   ArrayRef<llvm::Value *> getKnownExplosionVector() const {
      return Storage.get<ExplosionVector>(kind);
   }

   llvm::Value *getKnownSingletonExplosion() const {
      return Storage.get<SingletonExplosion>(kind);
   }

   const FunctionPointer &getFunctionPointer() const {
      return Storage.get<FunctionPointer>(kind);
   }
   // @todo
//   const ObjCMethod &getObjCMethod() const {
//      return Storage.get<ObjCMethod>(kind);
//   }

   const CoroutineState &getCoroutineState() const {
      return Storage.get<CoroutineState>(kind);
   }

   /// Produce an explosion for this lowered value.  Note that many
   /// different storage kinds can be turned into an explosion.
   Explosion getExplosion(IRGenFunction &IGF, PILType type) const {
      Explosion e;
      getExplosion(IGF, type, e);
      return e;
   }
   void getExplosion(IRGenFunction &IGF, PILType type, Explosion &ex) const;

   /// Produce an explosion which is known to be a single value.
   llvm::Value *getSingletonExplosion(IRGenFunction &IGF, PILType type) const;

   /// Produce a callee from this value.
   Callee getCallee(IRGenFunction &IGF, llvm::Value *selfValue,
                    CalleeInfo &&calleeInfo) const;
};

using PHINodeVector = llvm::TinyPtrVector<llvm::PHINode*>;

/// Represents a lowered PIL basic block. This keeps track
/// of PIL branch arguments so that they can be lowered to LLVM phi nodes.
struct LoweredBB {
   llvm::BasicBlock *bb;
   PHINodeVector phis;

   LoweredBB() = default;
   explicit LoweredBB(llvm::BasicBlock *bb, PHINodeVector &&phis)
      : bb(bb), phis(std::move(phis))
   {}
};

/// Visits a PIL Function and generates LLVM IR.
class IRGenPILFunction :
   public IRGenFunction, public PILInstructionVisitor<IRGenPILFunction>
{
public:
   llvm::DenseMap<PILValue, LoweredValue> LoweredValues;
   llvm::DenseMap<PILValue, StackAddress> LoweredPartialApplyAllocations;
   llvm::DenseMap<PILType, LoweredValue> LoweredUndefs;

   /// All alloc_ref instructions which allocate the object on the stack.
   llvm::SmallPtrSet<PILInstruction *, 8> StackAllocs;

   /// With closure captures it is actually possible to have two function
   /// arguments that both have the same name. Until this is fixed, we need to
   /// also hash the ArgNo here.
   using StackSlotKey =
   std::pair<unsigned, std::pair<const PILDebugScope *, StringRef>>;
   /// Keeps track of the mapping of source variables to -O0 shadow copy allocas.
   llvm::SmallDenseMap<StackSlotKey, Address, 8> ShadowStackSlots;
   llvm::SmallDenseMap<Decl *, SmallString<4>, 8> AnonymousVariables;
   /// To avoid inserting elements into ValueDomPoints twice.
   llvm::SmallDenseSet<llvm::Instruction *, 8> ValueVariables;
   /// Holds the DominancePoint of values that are storage for a source variable.
   SmallVector<std::pair<llvm::Instruction *, DominancePoint>, 8> ValueDomPoints;
   unsigned NumAnonVars = 0;

   /// Accumulative amount of allocated bytes on the stack. Used to limit the
   /// size for stack promoted objects.
   /// We calculate it on demand, so that we don't have to do it if the
   /// function does not have any stack promoted allocations.
   int EstimatedStackSize = -1;

   llvm::MapVector<PILBasicBlock *, LoweredBB> LoweredBBs;

   // Destination basic blocks for condfail traps.
   llvm::SmallVector<llvm::BasicBlock *, 8> FailBBs;

   PILFunction *CurPILFn;
   Address IndirectReturn;

   /// The unique block that calls @llvm.coro.end.
   llvm::BasicBlock *CoroutineExitBlock = nullptr;

   // A cached dominance analysis.
   std::unique_ptr<DominanceInfo> Dominance;

   IRGenPILFunction(IRGenModule &IGM, PILFunction *f);
   ~IRGenPILFunction();

   /// Generate IR for the PIL Function.
   void emitPILFunction();

   /// Calculates EstimatedStackSize.
   void estimateStackSize();

   void setLoweredValue(PILValue v, LoweredValue &&lv) {
      auto inserted = LoweredValues.insert({v, std::move(lv)});
      assert(inserted.second && "already had lowered value for sil value?!");
      (void)inserted;
   }

   /// Create a new Address corresponding to the given PIL address value.
   void setLoweredAddress(PILValue v, const Address &address) {
      assert(v->getType().isAddress() && "address for non-address value?!");
      setLoweredValue(v, address);
   }

   void setLoweredStackAddress(PILValue v, const StackAddress &address) {
      assert(v->getType().isAddress() && "address for non-address value?!");
      setLoweredValue(v, address);
   }

   void setLoweredDynamicallyEnforcedAddress(PILValue v,
                                             const Address &address,
                                             llvm::Value *scratch) {
      assert(v->getType().isAddress() && "address for non-address value?!");
      setLoweredValue(v, DynamicallyEnforcedAddress{address, scratch});
   }

   void setContainerOfUnallocatedAddress(PILValue v,
                                         const Address &buffer) {
      assert(v->getType().isAddress() && "address for non-address value?!");
      setLoweredValue(v,
                      LoweredValue(buffer, LoweredValue::ContainerForUnallocatedAddress));
   }

   void overwriteAllocatedAddress(PILValue v, const Address &address) {
      assert(v->getType().isAddress() && "address for non-address value?!");
      auto it = LoweredValues.find(v);
      assert(it != LoweredValues.end() && "no existing entry for overwrite?");
      assert(it->second.isUnallocatedAddressInBuffer() &&
             "not an unallocated address");
      it->second = ContainedAddress(it->second.getContainerOfAddress(), address);
   }

   void setAllocatedAddressForBuffer(PILValue v, const Address &allocedAddress);

   /// Create a new Explosion corresponding to the given PIL value.
   void setLoweredExplosion(PILValue v, Explosion &e) {
      assert(v->getType().isObject() && "explosion for address value?!");
      setLoweredValue(v, LoweredValue(e));
   }

   void setCorrespondingLoweredValues(PILInstructionResultArray results,
                                      Explosion &allValues) {
      for (PILValue result : results) {
         auto resultType = result->getType();
         auto &resultTI = getTypeInfo(resultType);

         // If the value is indirect, the next explosion value should just be
         // a pointer.
         if (resultType.isAddress()) {
            auto pointer = allValues.claimNext();
            setLoweredAddress(result, resultTI.getAddressForPointer(pointer));
            continue;
         }

         // Otherwise, claim out the right number of values.
         Explosion resultValue;
         cast<LoadableTypeInfo>(resultTI).reexplode(*this, allValues, resultValue);
         setLoweredExplosion(result, resultValue);
      }
   }

   void setLoweredBox(PILValue v, const OwnedAddress &box) {
      assert(v->getType().isObject() && "box for address value?!");
      setLoweredValue(v, LoweredValue(box));
   }

   /// Map the given PIL value to a FunctionPointer value.
   void setLoweredFunctionPointer(PILValue v, const FunctionPointer &fnPtr) {
      assert(v->getType().isObject() && "function for address value?!");
      assert(v->getType().is<PILFunctionType>() &&
             "function for non-function value?!");
      setLoweredValue(v, fnPtr);
   }

   /// Create a new Objective-C method corresponding to the given PIL value.
//   void setLoweredObjCMethod(PILValue v, PILDeclRef method) {
//      assert(v->getType().isObject() && "function for address value?!");
//      assert(v->getType().is<PILFunctionType>() &&
//             "function for non-function value?!");
//      setLoweredValue(v, ObjCMethod{method, PILType(), false});
//   }
   // @todo
   /// Create a new Objective-C method corresponding to the given PIL value that
   /// starts its search from the given search type.
   ///
   /// Unlike \c setLoweredObjCMethod, which finds the method in the actual
   /// runtime type of the object, this routine starts at the static type of the
   /// object and searches up the class hierarchy (toward superclasses).
   ///
   /// \param searchType The class from which the Objective-C runtime will start
   /// its search for a method.
   ///
   /// \param startAtSuper Whether we want to start at the superclass of the
   /// static type (vs. the static type itself).
//   void setLoweredObjCMethodBounded(PILValue v, PILDeclRef method,
//                                    PILType searchType, bool startAtSuper) {
//      assert(v->getType().isObject() && "function for address value?!");
//      assert(v->getType().is<PILFunctionType>() &&
//             "function for non-function value?!");
//      setLoweredValue(v, ObjCMethod{method, searchType, startAtSuper});
//   }

   void setLoweredCoroutine(PILValue tokenResult, CoroutineState &&state) {
      setLoweredValue(tokenResult, std::move(state));
   }

   LoweredValue &getUndefLoweredValue(PILType t) {
      auto found = LoweredUndefs.find(t);
      if (found != LoweredUndefs.end())
         return found->second;

      auto &ti = getTypeInfo(t);
      switch (t.getCategory()) {
         case PILValueCategory::Address: {
            Address undefAddr = ti.getAddressForPointer(
               llvm::UndefValue::get(ti.getStorageType()->getPointerTo()));
            LoweredUndefs.insert({t, LoweredValue(undefAddr)});
            break;
         }

         case PILValueCategory::Object: {
            auto schema = ti.getSchema();
            Explosion e;
            for (auto &elt : schema) {
               assert(!elt.isAggregate()
                      && "non-scalar element in loadable type schema?!");
               e.add(llvm::UndefValue::get(elt.getScalarType()));
            }
            LoweredUndefs.insert({t, LoweredValue(e)});
            break;
         }
      }

      found = LoweredUndefs.find(t);
      assert(found != LoweredUndefs.end());
      return found->second;
   }

   /// Get the LoweredValue corresponding to the given PIL value, which must
   /// have been lowered.
   LoweredValue &getLoweredValue(PILValue v) {
      if (isa<PILUndef>(v))
         return getUndefLoweredValue(v->getType());

      auto foundValue = LoweredValues.find(v);
      assert(foundValue != LoweredValues.end() &&
             "no lowered explosion for sil value!");
      return foundValue->second;
   }

   /// Get the Address of a PIL value of address type, which must have been
   /// lowered.
   Address getLoweredAddress(PILValue v) {
      return getLoweredValue(v).getAnyAddress();
   }

   StackAddress getLoweredStackAddress(PILValue v) {
      return getLoweredValue(v).getStackAddress();
   }

   llvm::Value *getLoweredDynamicEnforcementScratchBuffer(BeginAccessInst *v) {
      return getLoweredValue(v).getDynamicallyEnforcedAddress().ScratchBuffer;
   }

   const CoroutineState &getLoweredCoroutine(PILValue v) {
      return getLoweredValue(v).getCoroutineState();
   }

   /// Add the unmanaged LLVM values lowered from a PIL value to an explosion.
   void getLoweredExplosion(PILValue v, Explosion &e) {
      getLoweredValue(v).getExplosion(*this, v->getType(), e);
   }
   /// Create an Explosion containing the unmanaged LLVM values lowered from a
   /// PIL value.
   Explosion getLoweredExplosion(PILValue v) {
      return getLoweredValue(v).getExplosion(*this, v->getType());
   }

   /// Return the single member of the lowered explosion for the
   /// given PIL value.
   llvm::Value *getLoweredSingletonExplosion(PILValue v) {
      return getLoweredValue(v).getSingletonExplosion(*this, v->getType());
   }

   LoweredBB &getLoweredBB(PILBasicBlock *bb) {
      auto foundBB = LoweredBBs.find(bb);
      assert(foundBB != LoweredBBs.end() && "no llvm bb for sil bb?!");
      return foundBB->second;
   }

   TypeExpansionContext getExpansionContext() {
      return TypeExpansionContext(*CurPILFn);
   }

   PILType getLoweredTypeInContext(PILType ty) {
      return CurPILFn->getModule()
         .Types.getLoweredType(ty.getAstType(), getExpansionContext())
         .getCategoryType(ty.getCategory());
   }

   StringRef getOrCreateAnonymousVarName(VarDecl *Decl) {
      llvm::SmallString<4> &Name = AnonymousVariables[Decl];
      if (Name.empty()) {
         {
            llvm::raw_svector_ostream S(Name);
            S << '_' << NumAnonVars++;
         }
         AnonymousVariables.insert({Decl, Name});
      }
      return Name;
   }

   template <class DebugVarCarryingInst>
   StringRef getVarName(DebugVarCarryingInst *i, bool &IsAnonymous) {
      auto VarInfo = i->getVarInfo();
      if (!VarInfo)
         return StringRef();

      StringRef Name = i->getVarInfo()->Name;
      // The $match variables generated by the type checker are not
      // guaranteed to be unique within their scope, but they have
      // unique VarDecls.
      if ((Name.empty() || Name == "$match") && i->getDecl()) {
         IsAnonymous = true;
         return getOrCreateAnonymousVarName(i->getDecl());
      }
      return Name;
   }

   /// Try to emit an inline assembly gadget which extends the lifetime of
   /// \p Var. Returns whether or not this was successful.
   bool emitLifetimeExtendingUse(llvm::Value *Var) {
      llvm::Type *ArgTys;
      auto *Ty = Var->getType();
      // Vectors, Pointers and Floats are expected to fit into a register.
      if (Ty->isPointerTy() || Ty->isFloatingPointTy() || Ty->isVectorTy())
         ArgTys = {Ty};
      else {
         // If this is not a scalar or vector type, we can't handle it.
         if (isa<llvm::CompositeType>(Ty))
            return false;
         // The storage is guaranteed to be no larger than the register width.
         // Extend the storage so it would fit into a register.
         llvm::Type *IntTy;
         switch (IGM.getClangAstContext().getTargetInfo().getRegisterWidth()) {
            case 64:
               IntTy = IGM.Int64Ty;
               break;
            case 32:
               IntTy = IGM.Int32Ty;
               break;
            default:
               llvm_unreachable("unsupported register width");
         }
         ArgTys = {IntTy};
         Var = Builder.CreateZExtOrBitCast(Var, IntTy);
      }
      // Emit an empty inline assembler expression depending on the register.
      auto *AsmFnTy = llvm::FunctionType::get(IGM.VoidTy, ArgTys, false);
      auto *InlineAsm = llvm::InlineAsm::get(AsmFnTy, "", "r", true);
      Builder.CreateAsmCall(InlineAsm, Var);
      return true;
   }

   /// At -Onone, forcibly keep all LLVM values that are tracked by
   /// debug variables alive by inserting an empty inline assembler
   /// expression depending on the value in the blocks dominated by the
   /// value.
   void emitDebugVariableRangeExtension(const PILBasicBlock *CurBB) {
      if (IGM.IRGen.Opts.shouldOptimize())
         return;
      for (auto &Variable : ValueDomPoints) {
         llvm::Instruction *Var = Variable.first;
         DominancePoint VarDominancePoint = Variable.second;
         if (getActiveDominancePoint() == VarDominancePoint ||
             isActiveDominancePointDominatedBy(VarDominancePoint)) {
            bool ExtendedLifetime = emitLifetimeExtendingUse(Var);
            if (!ExtendedLifetime)
               continue;

            // Propagate dbg.values for Var into the current basic block. Note
            // that this shouldn't be necessary. LiveDebugValues should be doing
            // this but can't in general because it currently only tracks register
            // locations.
            llvm::BasicBlock *BB = Var->getParent();
            llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
            if (BB == CurBB)
               // The current basic block must be a successor of the dbg.value().
               continue;

            llvm::SmallVector<llvm::DbgValueInst *, 4> DbgValues;
            llvm::findDbgValues(DbgValues, Var);
            for (auto *DVI : DbgValues)
               if (DVI->getParent() == BB)
                  IGM.DebugInfo->getBuilder().insertDbgValueIntrinsic(
                     DVI->getValue(), DVI->getVariable(), DVI->getExpression(),
                     DVI->getDebugLoc(), &*CurBB->getFirstInsertionPt());
         }
      }
   }

   /// To make it unambiguous whether a `var` binding has been initialized,
   /// zero-initialize the shadow copy alloca. LLDB uses the first pointer-sized
   /// field to recognize to detect uninitizialized variables. This can be
   /// removed once swiftc switches to @llvm.dbg.addr() intrinsics.
   void zeroInit(llvm::AllocaInst *AI) {
      if (!AI)
         return;

      // Only do this at -Onone.
      uint64_t Size = *AI->getAllocationSizeInBits(IGM.DataLayout) / 8;
      if (IGM.IRGen.Opts.shouldOptimize() || !Size)
         return;

      llvm::IRBuilder<> ZeroInitBuilder(AI->getNextNode());

      // No debug location is how LLVM marks prologue instructions.
      ZeroInitBuilder.SetCurrentDebugLocation(nullptr);
      ZeroInitBuilder.CreateMemSet(
         AI, llvm::ConstantInt::get(IGM.Int8Ty, 0),
         Size, AI->getAlignment());
   }

   /// Account for bugs in LLVM.
   ///
   /// - The LLVM type legalizer currently doesn't update debug
   ///   intrinsics when a large value is split up into smaller
   ///   pieces. Note that this heuristic as a bit too conservative
   ///   on 32-bit targets as it will also fire for doubles.
   ///
   /// - CodeGen Prepare may drop dbg.values pointing to PHI instruction.
   bool needsShadowCopy(llvm::Value *Storage) {
      return (IGM.DataLayout.getTypeSizeInBits(Storage->getType()) >
              IGM.getClangAstContext().getTargetInfo().getRegisterWidth()) ||
             isa<llvm::PHINode>(Storage);
   }

   /// Unconditionally emit a stack shadow copy of an \c llvm::Value.
   llvm::Value *emitShadowCopy(llvm::Value *Storage, const PILDebugScope *Scope,
                               PILDebugVariable VarInfo, Alignment Align) {
      if (Align.isZero())
         Align = IGM.getPointerAlignment();

      unsigned ArgNo = VarInfo.ArgNo;
      auto &Alloca = ShadowStackSlots[{ArgNo, {Scope, VarInfo.Name}}];
      if (!Alloca.isValid())
         Alloca = createAlloca(Storage->getType(), Align, VarInfo.Name + ".debug");
      zeroInit(cast<llvm::AllocaInst>(Alloca.getAddress()));

      ArtificialLocation AutoRestore(Scope, IGM.DebugInfo.get(), Builder);
      Builder.CreateStore(Storage, Alloca.getAddress(), Align);
      return Alloca.getAddress();
   }

   /// At -Onone, emit a shadow copy of an Address in an alloca, so the
   /// register allocator doesn't elide the dbg.value intrinsic when
   /// register pressure is high.  There is a trade-off to this: With
   /// shadow copies, we lose the precise lifetime.
   llvm::Value *emitShadowCopyIfNeeded(llvm::Value *Storage,
                                       const PILDebugScope *Scope,
                                       PILDebugVariable VarInfo,
                                       bool IsAnonymous,
                                       Alignment Align = Alignment(0)) {
      // Never emit shadow copies when optimizing, or if already on the stack.
      // No debug info is emitted for refcounts either.
      if (IGM.IRGen.Opts.DisableDebuggerShadowCopies ||
          IGM.IRGen.Opts.shouldOptimize() || IsAnonymous ||
          isa<llvm::AllocaInst>(Storage) || isa<llvm::UndefValue>(Storage) ||
          Storage->getType() == IGM.RefCountedPtrTy)
         return Storage;

      // Always emit shadow copies for function arguments.
      if (VarInfo.ArgNo == 0)
         // Otherwise only if debug value range extension is not feasible.
         if (!needsShadowCopy(Storage)) {
            // Mark for debug value range extension unless this is a constant, or
            // unless it's not possible to emit lifetime-extending uses for this.
            if (auto *Value = dyn_cast<llvm::Instruction>(Storage)) {
               // Emit a use at the start of the storage lifetime to force early
               // materialization. This makes variables available for inspection as
               // soon as they are defined.
               bool ExtendedLifetime = emitLifetimeExtendingUse(Value);
               if (ExtendedLifetime)
                  if (ValueVariables.insert(Value).second)
                     ValueDomPoints.push_back({Value, getActiveDominancePoint()});
            }

            return Storage;
         }
      return emitShadowCopy(Storage, Scope, VarInfo, Align);
   }

   /// Like \c emitShadowCopyIfNeeded() but takes an \c Address instead of an
   /// \c llvm::Value.
   llvm::Value *emitShadowCopyIfNeeded(Address Storage,
                                       const PILDebugScope *Scope,
                                       PILDebugVariable VarInfo,
                                       bool IsAnonymous) {
      return emitShadowCopyIfNeeded(Storage.getAddress(), Scope, VarInfo,
                                    IsAnonymous, Storage.getAlignment());
   }

   /// Like \c emitShadowCopyIfNeeded() but takes an exploded value.
   void emitShadowCopyIfNeeded(PILValue &PILVal, const PILDebugScope *Scope,
                               PILDebugVariable VarInfo, bool IsAnonymous,
                               llvm::SmallVectorImpl<llvm::Value *> &copy) {
      Explosion e = getLoweredExplosion(PILVal);

      // Only do this at -O0.
      if (IGM.IRGen.Opts.DisableDebuggerShadowCopies ||
          IGM.IRGen.Opts.shouldOptimize() || IsAnonymous) {
         auto vals = e.claimAll();
         copy.append(vals.begin(), vals.end());
         return;
      }

      // Single or empty values.
      if (e.size() <= 1) {
         auto vals = e.claimAll();
         for (auto val : vals)
            copy.push_back(
               emitShadowCopyIfNeeded(val, Scope, VarInfo, IsAnonymous));
         return;
      }

      PILType Type = PILVal->getType();
      auto &LTI = cast<LoadableTypeInfo>(IGM.getTypeInfo(Type));
      auto Alloca = LTI.allocateStack(*this, Type, VarInfo.Name + ".debug");
      zeroInit(cast<llvm::AllocaInst>(Alloca.getAddress().getAddress()));
      ArtificialLocation AutoRestore(Scope, IGM.DebugInfo.get(), Builder);
      LTI.initialize(*this, e, Alloca.getAddress(), false /* isOutlined */);
      copy.push_back(Alloca.getAddressPointer());
   }

   /// Force all archetypes referenced by the type to be bound by this point.
   /// TODO: just make sure that we have a path to them that the debug info
   ///       can follow.
   void bindArchetypes(polar::Type Ty) {
      auto runtimeTy = IGM.getRuntimeReifiedType(Ty->getCanonicalType());
      if (!IGM.IRGen.Opts.shouldOptimize() && runtimeTy->hasArchetype())
         runtimeTy.visit([&](CanType t) {
            if (auto archetype = dyn_cast<ArchetypeType>(t))
               emitTypeMetadataRef(archetype);
         });
   }

   /// Emit debug info for a function argument or a local variable.
   template <typename StorageType>
   void emitDebugVariableDeclaration(StorageType Storage, DebugTypeInfo Ty,
                                     PILType PILTy, const PILDebugScope *DS,
                                     VarDecl *VarDecl, PILDebugVariable VarInfo,
                                     IndirectionKind Indirection = DirectValue) {
      assert(IGM.DebugInfo && "debug info not enabled");
      if (VarInfo.ArgNo) {
         PrologueLocation AutoRestore(IGM.DebugInfo.get(), Builder);
         IGM.DebugInfo->emitVariableDeclaration(Builder, Storage, Ty, DS, VarDecl,
                                                VarInfo, Indirection);
      } else
         IGM.DebugInfo->emitVariableDeclaration(Builder, Storage, Ty, DS, VarDecl,
                                                VarInfo, Indirection);
   }

   void emitFailBB() {
      if (!FailBBs.empty()) {
         // Move the trap basic blocks to the end of the function.
         for (auto *FailBB : FailBBs) {
            auto &BlockList = CurFn->getBasicBlockList();
            BlockList.splice(BlockList.end(), BlockList, FailBB);
         }
      }
   }

   //===--------------------------------------------------------------------===//
   // PIL instruction lowering
   //===--------------------------------------------------------------------===//

   void visitPILBasicBlock(PILBasicBlock *BB);

   void emitErrorResultVar(CanPILFunctionType FnTy,
                           PILResultInfo ErrorInfo,
                           DebugValueInst *DbgValue);
   void emitDebugInfoForAllocStack(AllocStackInst *i, const TypeInfo &type,
                                   llvm::Value *addr);
   void visitAllocStackInst(AllocStackInst *i);
   void visitAllocRefInst(AllocRefInst *i);
   void visitAllocRefDynamicInst(AllocRefDynamicInst *i);
   void visitAllocBoxInst(AllocBoxInst *i);

   void visitProjectBoxInst(ProjectBoxInst *i);

   void visitApplyInst(ApplyInst *i);
   void visitTryApplyInst(TryApplyInst *i);
   void visitFullApplySite(FullApplySite i);
   void visitPartialApplyInst(PartialApplyInst *i);
   void visitBuiltinInst(BuiltinInst *i);

   void visitFunctionRefBaseInst(FunctionRefBaseInst *i);
   void visitFunctionRefInst(FunctionRefInst *i);
   void visitDynamicFunctionRefInst(DynamicFunctionRefInst *i);
   void visitPreviousDynamicFunctionRefInst(PreviousDynamicFunctionRefInst *i);
   void visitAllocGlobalInst(AllocGlobalInst *i);
   void visitGlobalAddrInst(GlobalAddrInst *i);
   void visitGlobalValueInst(GlobalValueInst *i);

   void visitIntegerLiteralInst(IntegerLiteralInst *i);
   void visitFloatLiteralInst(FloatLiteralInst *i);
   void visitStringLiteralInst(StringLiteralInst *i);

   void visitLoadInst(LoadInst *i);
   void visitStoreInst(StoreInst *i);
   void visitAssignInst(AssignInst *i) {
      llvm_unreachable("assign is not valid in canonical PIL");
   }
   void visitAssignByWrapperInst(AssignByWrapperInst *i) {
      llvm_unreachable("assign_by_wrapper is not valid in canonical PIL");
   }
   void visitMarkUninitializedInst(MarkUninitializedInst *i) {
      llvm_unreachable("mark_uninitialized is not valid in canonical PIL");
   }
   void visitMarkFunctionEscapeInst(MarkFunctionEscapeInst *i) {
      llvm_unreachable("mark_function_escape is not valid in canonical PIL");
   }
   void visitLoadBorrowInst(LoadBorrowInst *i) {
      llvm_unreachable("unimplemented");
   }
   void visitDebugValueInst(DebugValueInst *i);
   void visitDebugValueAddrInst(DebugValueAddrInst *i);
   void visitRetainValueInst(RetainValueInst *i);
   void visitRetainValueAddrInst(RetainValueAddrInst *i);
   void visitCopyValueInst(CopyValueInst *i);
   void visitReleaseValueInst(ReleaseValueInst *i);
   void visitReleaseValueAddrInst(ReleaseValueAddrInst *i);
   void visitDestroyValueInst(DestroyValueInst *i);
   void visitAutoreleaseValueInst(AutoreleaseValueInst *i);
   void visitSetDeallocatingInst(SetDeallocatingInst *i);
   void visitObjectInst(ObjectInst *i)  {
      llvm_unreachable("object instruction cannot appear in a function");
   }
   void visitStructInst(StructInst *i);
   void visitTupleInst(TupleInst *i);
   void visitEnumInst(EnumInst *i);
   void visitInitEnumDataAddrInst(InitEnumDataAddrInst *i);
   void visitSelectEnumInst(SelectEnumInst *i);
   void visitSelectEnumAddrInst(SelectEnumAddrInst *i);
   void visitSelectValueInst(SelectValueInst *i);
   void visitUncheckedEnumDataInst(UncheckedEnumDataInst *i);
   void visitUncheckedTakeEnumDataAddrInst(UncheckedTakeEnumDataAddrInst *i);
   void visitInjectEnumAddrInst(InjectEnumAddrInst *i);
//   void visitObjCInterfaceInst(ObjCInterfaceInst *i);
   void visitMetatypeInst(MetatypeInst *i);
   void visitValueMetatypeInst(ValueMetatypeInst *i);
   void visitExistentialMetatypeInst(ExistentialMetatypeInst *i);
   void visitTupleExtractInst(TupleExtractInst *i);
   void visitDestructureTupleInst(DestructureTupleInst *i) {
      llvm_unreachable("unimplemented");
   }
   void visitDestructureStructInst(DestructureStructInst *i) {
      llvm_unreachable("unimplemented");
   }
   void visitTupleElementAddrInst(TupleElementAddrInst *i);
   void visitStructExtractInst(StructExtractInst *i);
   void visitStructElementAddrInst(StructElementAddrInst *i);
   void visitRefElementAddrInst(RefElementAddrInst *i);
   void visitRefTailAddrInst(RefTailAddrInst *i);

   void visitClassMethodInst(ClassMethodInst *i);
   void visitSuperMethodInst(SuperMethodInst *i);
   void visitObjCMethodInst(ObjCMethodInst *i);
   void visitObjCSuperMethodInst(ObjCSuperMethodInst *i);
   void visitWitnessMethodInst(WitnessMethodInst *i);

   void visitAllocValueBufferInst(AllocValueBufferInst *i);
   void visitProjectValueBufferInst(ProjectValueBufferInst *i);
   void visitDeallocValueBufferInst(DeallocValueBufferInst *i);

   void visitOpenExistentialAddrInst(OpenExistentialAddrInst *i);
   void visitOpenExistentialMetatypeInst(OpenExistentialMetatypeInst *i);
   void visitOpenExistentialRefInst(OpenExistentialRefInst *i);
   void visitOpenExistentialValueInst(OpenExistentialValueInst *i);
   void visitInitExistentialAddrInst(InitExistentialAddrInst *i);
   void visitInitExistentialValueInst(InitExistentialValueInst *i);
   void visitInitExistentialMetatypeInst(InitExistentialMetatypeInst *i);
   void visitInitExistentialRefInst(InitExistentialRefInst *i);
   void visitDeinitExistentialAddrInst(DeinitExistentialAddrInst *i);
   void visitDeinitExistentialValueInst(DeinitExistentialValueInst *i);

   void visitAllocExistentialBoxInst(AllocExistentialBoxInst *i);
   void visitOpenExistentialBoxInst(OpenExistentialBoxInst *i);
   void visitOpenExistentialBoxValueInst(OpenExistentialBoxValueInst *i);
   void visitProjectExistentialBoxInst(ProjectExistentialBoxInst *i);
   void visitDeallocExistentialBoxInst(DeallocExistentialBoxInst *i);

   void visitProjectBlockStorageInst(ProjectBlockStorageInst *i);
   void visitInitBlockStorageHeaderInst(InitBlockStorageHeaderInst *i);

   void visitFixLifetimeInst(FixLifetimeInst *i);
   void visitEndLifetimeInst(EndLifetimeInst *i) {
      llvm_unreachable("unimplemented");
   }
   void
   visitUncheckedOwnershipConversionInst(UncheckedOwnershipConversionInst *i) {
      llvm_unreachable("unimplemented");
   }
   void visitBeginBorrowInst(BeginBorrowInst *i) {
      llvm_unreachable("unimplemented");
   }
   void visitEndBorrowInst(EndBorrowInst *i) {
      llvm_unreachable("unimplemented");
   }
   void visitStoreBorrowInst(StoreBorrowInst *i) {
      llvm_unreachable("unimplemented");
   }
   void visitBeginAccessInst(BeginAccessInst *i);
   void visitEndAccessInst(EndAccessInst *i);
   void visitBeginUnpairedAccessInst(BeginUnpairedAccessInst *i);
   void visitEndUnpairedAccessInst(EndUnpairedAccessInst *i);
   void visitUnmanagedRetainValueInst(UnmanagedRetainValueInst *i) {
      llvm_unreachable("unimplemented");
   }
   void visitUnmanagedReleaseValueInst(UnmanagedReleaseValueInst *i) {
      llvm_unreachable("unimplemented");
   }
   void visitUnmanagedAutoreleaseValueInst(UnmanagedAutoreleaseValueInst *i) {
      llvm_unreachable("unimplemented");
   }
   void visitMarkDependenceInst(MarkDependenceInst *i);
   void visitCopyBlockInst(CopyBlockInst *i);
   void visitCopyBlockWithoutEscapingInst(CopyBlockWithoutEscapingInst *i) {
      llvm_unreachable("not valid in canonical PIL");
   }
   void visitStrongRetainInst(StrongRetainInst *i);
   void visitStrongReleaseInst(StrongReleaseInst *i);
   void visitIsUniqueInst(IsUniqueInst *i);
   void visitIsEscapingClosureInst(IsEscapingClosureInst *i);
   void visitDeallocStackInst(DeallocStackInst *i);
   void visitDeallocBoxInst(DeallocBoxInst *i);
   void visitDeallocRefInst(DeallocRefInst *i);
   void visitDeallocPartialRefInst(DeallocPartialRefInst *i);

   void visitCopyAddrInst(CopyAddrInst *i);
   void visitDestroyAddrInst(DestroyAddrInst *i);

   void visitBindMemoryInst(BindMemoryInst *i);

   void visitCondFailInst(CondFailInst *i);

   void visitConvertFunctionInst(ConvertFunctionInst *i);
   void visitConvertEscapeToNoEscapeInst(ConvertEscapeToNoEscapeInst *i);
   void visitThinFunctionToPointerInst(ThinFunctionToPointerInst *i);
   void visitPointerToThinFunctionInst(PointerToThinFunctionInst *i);
   void visitUpcastInst(UpcastInst *i);
   void visitAddressToPointerInst(AddressToPointerInst *i);
   void visitPointerToAddressInst(PointerToAddressInst *i);
   void visitUncheckedRefCastInst(UncheckedRefCastInst *i);
   void visitUncheckedRefCastAddrInst(UncheckedRefCastAddrInst *i);
   void visitUncheckedAddrCastInst(UncheckedAddrCastInst *i);
   void visitUncheckedTrivialBitCastInst(UncheckedTrivialBitCastInst *i);
   void visitUncheckedBitwiseCastInst(UncheckedBitwiseCastInst *i);
   void visitRefToRawPointerInst(RefToRawPointerInst *i);
   void visitRawPointerToRefInst(RawPointerToRefInst *i);
   void visitThinToThickFunctionInst(ThinToThickFunctionInst *i);
//   void visitThickToObjCMetatypeInst(ThickToObjCMetatypeInst *i);
//   void visitObjCToThickMetatypeInst(ObjCToThickMetatypeInst *i);
   void visitUnconditionalCheckedCastInst(UnconditionalCheckedCastInst *i);
   void visitUnconditionalCheckedCastAddrInst(UnconditionalCheckedCastAddrInst *i);
   void
   visitUnconditionalCheckedCastValueInst(UnconditionalCheckedCastValueInst *i);
//   void visitObjCMetatypeToObjectInst(ObjCMetatypeToObjectInst *i);
//   void visitObjCExistentialMetatypeToObjectInst(
//      ObjCExistentialMetatypeToObjectInst *i);
   void visitRefToBridgeObjectInst(RefToBridgeObjectInst *i);
   void visitClassifyBridgeObjectInst(ClassifyBridgeObjectInst *i);
   void visitBridgeObjectToRefInst(BridgeObjectToRefInst *i);
   void visitBridgeObjectToWordInst(BridgeObjectToWordInst *i);
   void visitValueToBridgeObjectInst(ValueToBridgeObjectInst *i);

   void visitIndexAddrInst(IndexAddrInst *i);
   void visitTailAddrInst(TailAddrInst *i);
   void visitIndexRawPointerInst(IndexRawPointerInst *i);

   void visitBeginApplyInst(BeginApplyInst *i);
   void visitEndApplyInst(EndApplyInst *i);
   void visitAbortApplyInst(AbortApplyInst *i);
   void visitEndApply(BeginApplyInst *i, bool isAbort);

   void visitUnreachableInst(UnreachableInst *i);
   void visitBranchInst(BranchInst *i);
   void visitCondBranchInst(CondBranchInst *i);
   void visitReturnInst(ReturnInst *i);
   void visitThrowInst(ThrowInst *i);
   void visitUnwindInst(UnwindInst *i);
   void visitYieldInst(YieldInst *i);
   void visitSwitchValueInst(SwitchValueInst *i);
   void visitSwitchEnumInst(SwitchEnumInst *i);
   void visitSwitchEnumAddrInst(SwitchEnumAddrInst *i);
   void visitDynamicMethodBranchInst(DynamicMethodBranchInst *i);
   void visitCheckedCastBranchInst(CheckedCastBranchInst *i);
   void visitCheckedCastValueBranchInst(CheckedCastValueBranchInst *i);
   void visitCheckedCastAddrBranchInst(CheckedCastAddrBranchInst *i);

   void visitKeyPathInst(KeyPathInst *I);

#define LOADABLE_REF_STORAGE_HELPER(Name)                                      \
  void visitRefTo##Name##Inst(RefTo##Name##Inst *i);                           \
  void visit##Name##ToRefInst(Name##ToRefInst *i);                             \
  void visitStrongCopy##Name##ValueInst(StrongCopy##Name##ValueInst *i);
#define NEVER_LOADABLE_CHECKED_REF_STORAGE(Name, ...) \
  void visitLoad##Name##Inst(Load##Name##Inst *i); \
  void visitStore##Name##Inst(Store##Name##Inst *i);
#define ALWAYS_LOADABLE_CHECKED_REF_STORAGE(Name, ...)                         \
  LOADABLE_REF_STORAGE_HELPER(Name)                                            \
  void visitStrongRetain##Name##Inst(StrongRetain##Name##Inst *i);             \
  void visit##Name##RetainInst(Name##RetainInst *i);                           \
  void visit##Name##ReleaseInst(Name##ReleaseInst *i);
#define SOMETIMES_LOADABLE_CHECKED_REF_STORAGE(Name, ...) \
  NEVER_LOADABLE_CHECKED_REF_STORAGE(Name, "...") \
  ALWAYS_LOADABLE_CHECKED_REF_STORAGE(Name, "...")
#define UNCHECKED_REF_STORAGE(Name, ...) \
  LOADABLE_REF_STORAGE_HELPER(Name)
#include "polarphp/ast/ReferenceStorageDef.h"
#undef LOADABLE_REF_STORAGE_HELPER
};

} // end anonymous namespace

void LoweredValue::getExplosion(IRGenFunction &IGF, PILType type,
                                Explosion &ex) const {
   switch (kind) {
      case Kind::StackAddress:
      case Kind::ContainedAddress:
      case Kind::DynamicallyEnforcedAddress:
      case Kind::CoroutineState:
         llvm_unreachable("not a value");

      case Kind::ExplosionVector:
         ex.add(Storage.get<ExplosionVector>(kind));
         return;

      case Kind::SingletonExplosion:
         ex.add(Storage.get<SingletonExplosion>(kind));
         return;

      case Kind::EmptyExplosion:
         return;

      case Kind::OwnedAddress:
         ex.add(Storage.get<OwnedAddress>(kind).getOwner());
         return;

      case Kind::FunctionPointer:
         ex.add(Storage.get<FunctionPointer>(kind)
                   .getExplosionValue(IGF, type.castTo<PILFunctionType>()));
         return;

//      case Kind::ObjCMethod:
//         ex.add(Storage.get<ObjCMethod>(kind).getExplosionValue(IGF));
//         return;
   }
   llvm_unreachable("bad kind");
}

llvm::Value *LoweredValue::getSingletonExplosion(IRGenFunction &IGF,
                                                 PILType type) const {
   switch (kind) {
      case Kind::StackAddress:
      case Kind::ContainedAddress:
      case Kind::DynamicallyEnforcedAddress:
      case Kind::CoroutineState:
         llvm_unreachable("not a value");

      case Kind::EmptyExplosion:
      case Kind::ExplosionVector:
         llvm_unreachable("not a singleton explosion");

      case Kind::SingletonExplosion:
         return Storage.get<SingletonExplosion>(kind);

      case Kind::OwnedAddress:
         return Storage.get<OwnedAddress>(kind).getOwner();

      case Kind::FunctionPointer:
         return Storage.get<FunctionPointer>(kind)
            .getExplosionValue(IGF, type.castTo<PILFunctionType>());
         // @todo
//      case Kind::ObjCMethod:
//         return Storage.get<ObjCMethod>(kind).getExplosionValue(IGF);
   }
   llvm_unreachable("bad kind");
}

IRGenPILFunction::IRGenPILFunction(IRGenModule &IGM, PILFunction *f)
   : IRGenFunction(IGM,
                   IGM.getAddrOfPILFunction(f, ForDefinition,
                                            f->isDynamicallyReplaceable()),
                   f->getOptimizationMode(), f->getDebugScope(),
                   f->getLocation()),
     CurPILFn(f) {
   // Apply sanitizer attributes to the function.
   // TODO: Check if the function is supposed to be excluded from ASan either by
   // being in the external file or via annotations.
   if (IGM.IRGen.Opts.Sanitizers & SanitizerKind::Address)
      CurFn->addFnAttr(llvm::Attribute::SanitizeAddress);
   if (IGM.IRGen.Opts.Sanitizers & SanitizerKind::Thread) {
      auto declContext = f->getDeclContext();
      if (declContext && isa<DestructorDecl>(declContext)) {
         // Do not report races in deinit and anything called from it
         // because TSan does not observe synchronization between retain
         // count dropping to '0' and the object deinitialization.
         CurFn->addFnAttr("sanitize_thread_no_checking_at_run_time");
      } else {
         CurFn->addFnAttr(llvm::Attribute::SanitizeThread);
      }
   }

   // Disable inlining of coroutine functions until we split.
   if (f->getLoweredFunctionType()->isCoroutine()) {
      CurFn->addFnAttr(llvm::Attribute::NoInline);
   }
   // Emit the thunk that calls the previous implementation if this is a dynamic
   // replacement.
   if (f->getDynamicallyReplacedFunction()) {
      IGM.emitDynamicReplacementOriginalFunctionThunk(f);
   }

   if (f->isDynamicallyReplaceable()) {
      IGM.createReplaceableProlog(*this, f);
   }
}

IRGenPILFunction::~IRGenPILFunction() {
   assert(Builder.hasPostTerminatorIP() && "did not terminate BB?!");
   // Emit the fail BB if we have one.
   if (!FailBBs.empty())
      emitFailBB();
   LLVM_DEBUG(CurFn->print(llvm::dbgs()));
}

template<typename ValueVector>
static void emitPHINodesForType(IRGenPILFunction &IGF, PILType type,
                                const TypeInfo &ti, unsigned predecessors,
                                ValueVector &phis) {
   if (type.isAddress()) {
      phis.push_back(IGF.Builder.CreatePHI(ti.getStorageType()->getPointerTo(),
                                           predecessors));
   } else {
      // PHIs are always emitted with maximal explosion.
      ExplosionSchema schema = ti.getSchema();
      for (auto &elt : schema) {
         if (elt.isScalar())
            phis.push_back(
               IGF.Builder.CreatePHI(elt.getScalarType(), predecessors));
         else
            phis.push_back(
               IGF.Builder.CreatePHI(elt.getAggregateType()->getPointerTo(),
                                     predecessors));
      }
   }
}

static PHINodeVector
emitPHINodesForBBArgs(IRGenPILFunction &IGF,
                      PILBasicBlock *silBB,
                      llvm::BasicBlock *llBB) {
   PHINodeVector phis;
   unsigned predecessors = std::distance(silBB->pred_begin(), silBB->pred_end());

   IGF.Builder.SetInsertPoint(llBB);
   if (IGF.IGM.DebugInfo) {
      // Use the location of the first instruction in the basic block
      // for the φ-nodes.
      if (!silBB->empty()) {
         PILInstruction &I = *silBB->begin();
         auto DS = I.getDebugScope();
         assert(DS);
         IGF.IGM.DebugInfo->setCurrentLoc(IGF.Builder, DS, I.getLoc());
      }
   }

   for (PILArgument *arg : make_range(silBB->args_begin(), silBB->args_end())) {
      size_t first = phis.size();

      const TypeInfo &ti = IGF.getTypeInfo(arg->getType());

      emitPHINodesForType(IGF, arg->getType(), ti, predecessors, phis);
      if (arg->getType().isAddress()) {
         IGF.setLoweredAddress(arg,
                               ti.getAddressForPointer(phis.back()));
      } else {
         Explosion argValue;
         for (llvm::PHINode *phi :
            polar::make_range(phis.begin()+first, phis.end()))
            argValue.add(phi);
         IGF.setLoweredExplosion(arg, argValue);
      }
   }

   // Since we return to the entry of the function, reset the location.
   if (IGF.IGM.DebugInfo)
      IGF.IGM.DebugInfo->clearLoc(IGF.Builder);

   return phis;
}

static void addIncomingExplosionToPHINodes(IRGenPILFunction &IGF,
                                           LoweredBB &lbb,
                                           unsigned &phiIndex,
                                           Explosion &argValue);

// TODO: Handle this during PIL AddressLowering.
static ArrayRef<PILArgument*> emitEntryPointIndirectReturn(
   IRGenPILFunction &IGF,
   PILBasicBlock *entry,
   Explosion &params,
   CanPILFunctionType funcTy,
   llvm::function_ref<bool(PILType)> requiresIndirectResult) {
   // Map an indirect return for a type PIL considers loadable but still
   // requires an indirect return at the IR level.
   PILFunctionConventions fnConv(funcTy, IGF.getPILModule());
   PILType directResultType =
      IGF.CurPILFn->mapTypeIntoContext(fnConv.getPILResultType());
   if (requiresIndirectResult(directResultType)) {
      auto &paramTI = IGF.IGM.getTypeInfo(directResultType);
      auto &retTI =
         IGF.IGM.getTypeInfo(IGF.getLoweredTypeInContext(directResultType));
      auto ptr = params.claimNext();
      if (paramTI.getStorageType() != retTI.getStorageType()) {
         assert(directResultType.getAstType()->hasOpaqueArchetype());
         ptr = IGF.Builder.CreateBitCast(ptr,
                                         retTI.getStorageType()->getPointerTo());
      }
      IGF.IndirectReturn = retTI.getAddressForPointer(ptr);
   }

   auto bbargs = entry->getArguments();

   // Map the indirect returns if present.
   unsigned numIndirectResults = fnConv.getNumIndirectPILResults();
   unsigned idx = 0;
   for (auto indirectResultType : fnConv.getIndirectPILResultTypes()) {
      PILArgument *ret = bbargs[idx];
      auto inContextResultType =
         IGF.CurPILFn->mapTypeIntoContext(indirectResultType);
      auto &retTI = IGF.IGM.getTypeInfo(ret->getType());
      // The parameter's type might be different due to looking through opaque
      // archetypes.
      auto ptr = params.claimNext();
      auto &paramTI = IGF.IGM.getTypeInfo(inContextResultType);
      if (paramTI.getStorageType() != retTI.getStorageType()) {
         assert(inContextResultType.getAstType()->hasOpaqueArchetype());
         ptr = IGF.Builder.CreateBitCast(ptr,
                                         retTI.getStorageType()->getPointerTo());
      }
      auto addr = retTI.getAddressForPointer(ptr);
      IGF.setLoweredAddress(ret, addr);
      ++idx;
   }
   assert(numIndirectResults == idx);

   return bbargs.slice(numIndirectResults);
}

static void bindParameter(IRGenPILFunction &IGF,
                          PILArgument *param,
                          PILType paramTy,
                          Explosion &allParamValues) {
   // Pull out the parameter value and its formal type.
   auto &paramTI = IGF.getTypeInfo(IGF.CurPILFn->mapTypeIntoContext(paramTy));
   auto &argTI = IGF.getTypeInfo(param->getType());

   // If the PIL parameter isn't passed indirectly, we need to map it
   // to an explosion.
   if (param->getType().isObject()) {
      Explosion paramValues;
      auto &loadableParamTI = cast<LoadableTypeInfo>(paramTI);
      auto &loadableArgTI = cast<LoadableTypeInfo>(paramTI);
      // If the explosion must be passed indirectly, load the value from the
      // indirect address.
      auto &nativeSchema = argTI.nativeParameterValueSchema(IGF.IGM);
      if (nativeSchema.requiresIndirect()) {
         Address paramAddr =
            loadableParamTI.getAddressForPointer(allParamValues.claimNext());
         if (paramTI.getStorageType() != argTI.getStorageType())
            paramAddr =
               loadableArgTI.getAddressForPointer(IGF.Builder.CreateBitCast(
                  paramAddr.getAddress(),
                  loadableArgTI.getStorageType()->getPointerTo()));
         loadableArgTI.loadAsTake(IGF, paramAddr, paramValues);
      } else {
         if (!nativeSchema.empty()) {
            // Otherwise, we map from the native convention to the type's explosion
            // schema.
            Explosion nativeParam;
            allParamValues.transferInto(nativeParam, nativeSchema.size());
            paramValues = nativeSchema.mapFromNative(IGF.IGM, IGF, nativeParam,
                                                     param->getType());
         } else {
            assert(paramTI.getSchema().empty());
         }
      }
      IGF.setLoweredExplosion(param, paramValues);
      return;
   }

   // Okay, the type is passed indirectly in PIL, so we need to map
   // it to an address.
   // FIXME: that doesn't mean we should physically pass it
   // indirectly at this resilience expansion. An @in or @in_guaranteed parameter
   // could be passed by value in the right resilience domain.
   auto ptr = allParamValues.claimNext();
   if (paramTI.getStorageType() != argTI.getStorageType()) {
      ptr =
         IGF.Builder.CreateBitCast(ptr, argTI.getStorageType()->getPointerTo());
   }
   Address paramAddr = argTI.getAddressForPointer(ptr);
   IGF.setLoweredAddress(param, paramAddr);
}

/// Emit entry point arguments for a PILFunction with the Swift calling
/// convention.
static void emitEntryPointArgumentsNativeCC(IRGenPILFunction &IGF,
                                            PILBasicBlock *entry,
                                            Explosion &allParamValues) {
   auto funcTy = IGF.CurPILFn->getLoweredFunctionType();

   // Map the indirect return if present.
   ArrayRef<PILArgument *> params = emitEntryPointIndirectReturn(
      IGF, entry, allParamValues, funcTy, [&](PILType retType) -> bool {
         auto &schema =
            IGF.IGM.getTypeInfo(retType).nativeReturnValueSchema(IGF.IGM);
         return schema.requiresIndirect();
      });

   // The witness method CC passes Self as a final argument.
   WitnessMetadata witnessMetadata;
   if (funcTy->getRepresentation() == PILFunctionTypeRepresentation::WitnessMethod) {
      collectTrailingWitnessMetadata(IGF, *IGF.CurPILFn, allParamValues,
                                     witnessMetadata);
   }

   // Bind the error result by popping it off the parameter list.
   if (funcTy->hasErrorResult()) {
      IGF.setErrorResultSlot(allParamValues.takeLast());
   }
   // The coroutine context should be the first parameter.
   switch (funcTy->getCoroutineKind()) {
      case PILCoroutineKind::None:
         break;
      case PILCoroutineKind::YieldOnce:
         emitYieldOnceCoroutineEntry(IGF, funcTy, allParamValues);
         break;
      case PILCoroutineKind::YieldMany:
         emitYieldManyCoroutineEntry(IGF, funcTy, allParamValues);
         break;
   }

   PILFunctionConventions conv(funcTy, IGF.getPILModule());

   // The 'self' argument might be in the context position, which is
   // now the end of the parameter list.  Bind it now.
   if (hasSelfContextParameter(funcTy)) {
      PILArgument *selfParam = params.back();
      params = params.drop_back();

      Explosion selfTemp;
      selfTemp.add(allParamValues.takeLast());
      bindParameter(IGF, selfParam,
                    conv.getPILArgumentType(conv.getNumPILArguments() - 1),
                    selfTemp);

      // Even if we don't have a 'self', if we have an error result, we
      // should have a placeholder argument here.
   } else if (funcTy->hasErrorResult() ||
              funcTy->getRepresentation() == PILFunctionTypeRepresentation::Thick)
   {
      llvm::Value *contextPtr = allParamValues.takeLast(); (void) contextPtr;
      assert(contextPtr->getType() == IGF.IGM.RefCountedPtrTy);
   }

   // Map the remaining PIL parameters to LLVM parameters.
   unsigned i = 0;
   for (PILArgument *param : params) {
      auto argIdx = conv.getPILArgIndexOfFirstParam() + i;
      bindParameter(IGF, param, conv.getPILArgumentType(argIdx), allParamValues);
      ++i;
   }

   // Bind polymorphic arguments.  This can only be done after binding
   // all the value parameters.
   if (hasPolymorphicParameters(funcTy)) {
      emitPolymorphicParameters(IGF, *IGF.CurPILFn, allParamValues,
                                &witnessMetadata,
                                [&](unsigned paramIndex) -> llvm::Value* {
                                   PILValue parameter =
                                      IGF.CurPILFn->getArgumentsWithoutIndirectResults()[paramIndex];
                                   return IGF.getLoweredSingletonExplosion(parameter);
                                });
   }

   assert(allParamValues.empty() && "didn't claim all parameters!");
}

/// Emit entry point arguments for the parameters of a C function, or the
/// method parameters of an ObjC method.
static void emitEntryPointArgumentsCOrObjC(IRGenPILFunction &IGF,
                                           PILBasicBlock *entry,
                                           Explosion &params,
                                           CanPILFunctionType funcTy) {
   // First, lower the method type.
   ForeignFunctionInfo foreignInfo = IGF.IGM.getForeignFunctionInfo(funcTy);
   assert(foreignInfo.ClangInfo);
   auto &FI = *foreignInfo.ClangInfo;

   // Okay, start processing the parameters explosion.

   // First, claim all the indirect results.
   ArrayRef<PILArgument*> args
      = emitEntryPointIndirectReturn(IGF, entry, params, funcTy,
                                     [&](PILType directResultType) -> bool {
                                        return FI.getReturnInfo().isIndirect();
                                     });

   unsigned nextArgTyIdx = 0;

   // Handle the arguments of an ObjC method.
   if (IGF.CurPILFn->getRepresentation() ==
       PILFunctionTypeRepresentation::ObjCMethod) {
      // Claim the self argument from the end of the formal arguments.
      PILArgument *selfArg = args.back();
      args = args.slice(0, args.size() - 1);

      // Set the lowered explosion for the self argument.
      auto &selfTI = cast<LoadableTypeInfo>(IGF.getTypeInfo(selfArg->getType()));
      auto selfSchema = selfTI.getSchema();
      assert(selfSchema.size() == 1 && "Expected self to be a single element!");

      auto *selfValue = params.claimNext();
      auto *bodyType = selfSchema.begin()->getScalarType();
      if (selfValue->getType() != bodyType)
         selfValue = IGF.coerceValue(selfValue, bodyType, IGF.IGM.DataLayout);

      Explosion self;
      self.add(selfValue);
      IGF.setLoweredExplosion(selfArg, self);

      // Discard the implicit _cmd argument.
      params.claimNext();

      // We've handled the self and _cmd arguments, so when we deal with
      // generating explosions for the remaining arguments we can skip
      // these.
      nextArgTyIdx = 2;
   }

   assert(args.size() == (FI.arg_size() - nextArgTyIdx) &&
          "Number of arguments not equal to number of argument types!");

   // Generate lowered explosions for each explicit argument.
   for (auto i : indices(args)) {
      PILArgument *arg = args[i];
      auto argTyIdx = i + nextArgTyIdx;
      auto &argTI = IGF.getTypeInfo(arg->getType());

      // Bitcast indirect argument pointers to the right storage type.
      if (arg->getType().isAddress()) {
         llvm::Value *ptr = params.claimNext();
         ptr = IGF.Builder.CreateBitCast(ptr,
                                         argTI.getStorageType()->getPointerTo());
         IGF.setLoweredAddress(arg, Address(ptr, argTI.getBestKnownAlignment()));
         continue;
      }

      auto &loadableArgTI = cast<LoadableTypeInfo>(argTI);

      Explosion argExplosion;
      emitForeignParameter(IGF, params, foreignInfo, argTyIdx, arg->getType(),
                           loadableArgTI, argExplosion, false);
      IGF.setLoweredExplosion(arg, argExplosion);
   }

   assert(params.empty() && "didn't claim all parameters!");

   // emitPolymorphicParameters() may create function calls, so we need
   // to initialize the debug location here.
   ArtificialLocation Loc(IGF.getDebugScope(), IGF.IGM.DebugInfo.get(),
                          IGF.Builder);

   // Bind polymorphic arguments. This can only be done after binding
   // all the value parameters, and must be done even for non-polymorphic
   // functions because of imported Objective-C generics.
   emitPolymorphicParameters(
      IGF, *IGF.CurPILFn, params, nullptr,
      [&](unsigned paramIndex) -> llvm::Value * {
         PILValue parameter = entry->getArguments()[paramIndex];
         return IGF.getLoweredSingletonExplosion(parameter);
      });
}

/// Get metadata for the dynamic Self type if we have it.
static void emitLocalSelfMetadata(IRGenPILFunction &IGF) {
   if (!IGF.CurPILFn->hasSelfMetadataParam())
      return;

   const PILArgument *selfArg = IGF.CurPILFn->getSelfMetadataArgument();
   auto selfTy = selfArg->getType().getAstType();
   CanMetatypeType metaTy =
      dyn_cast<MetatypeType>(selfTy);
   IRGenFunction::LocalSelfKind selfKind;
   if (!metaTy)
      selfKind = IRGenFunction::ObjectReference;
   else {
      selfTy = metaTy.getInstanceType();
      switch (metaTy->getRepresentation()) {
         case MetatypeRepresentation::Thin:
            llvm_unreachable("class metatypes are never thin");
         case MetatypeRepresentation::Thick:
            selfKind = IRGenFunction::SwiftMetatype;
            break;
         case MetatypeRepresentation::ObjC:
            selfKind = IRGenFunction::ObjCMetatype;
            break;
      }
   }
   llvm::Value *value = IGF.getLoweredExplosion(selfArg).claimNext();
   if (auto dynSelfTy = dyn_cast<DynamicSelfType>(selfTy))
      selfTy = dynSelfTy.getSelfType();

   // Specify the exact Self type if we know it, either because the class
   // is final, or because the function we're emitting is a method with the
   // [exact_self_class] attribute set on it during the PIL pipeline.
   bool isExact = selfTy->getClassOrBoundGenericClass()->isFinal()
                  || IGF.CurPILFn->isExactSelfClass();

   IGF.setLocalSelfMetadata(selfTy, isExact, value, selfKind);
}

/// Emit the definition for the given PIL constant.
void IRGenModule::emitPILFunction(PILFunction *f) {
   if (f->isExternalDeclaration())
      return;

   // Do not emit bodies of public_external functions.
   if (hasPublicVisibility(f->getLinkage()) && f->isAvailableExternally())
      return;

   PrettyStackTracePILFunction stackTrace("emitting IR", f);
   llvm::SaveAndRestore<SourceFile *> SetCurSourceFile(CurSourceFile);
   if (auto dc = f->getModule().getAssociatedContext()) {
      if (auto sf = dc->getParentSourceFile()) {
         CurSourceFile = sf;
      }
   }
   IRGenPILFunction(*this, f).emitPILFunction();
}

void IRGenPILFunction::emitPILFunction() {
   LLVM_DEBUG(llvm::dbgs() << "emitting PIL function: ";
                 CurPILFn->printName(llvm::dbgs());
                 llvm::dbgs() << '\n';
                 CurPILFn->print(llvm::dbgs()));

   assert(!CurPILFn->empty() && "function has no basic blocks?!");

   if (CurPILFn->getDynamicallyReplacedFunction())
      IGM.IRGen.addDynamicReplacement(CurPILFn);

   // Configure the dominance resolver.
   // TODO: consider re-using a dom analysis from the PassManager
   // TODO: consider using a cheaper analysis at -O0
   setDominanceResolver([](IRGenFunction &IGF_,
                           DominancePoint activePoint,
                           DominancePoint dominatingPoint) -> bool {
      IRGenPILFunction &IGF = static_cast<IRGenPILFunction&>(IGF_);
      if (!IGF.Dominance) {
         IGF.Dominance.reset(new DominanceInfo(IGF.CurPILFn));
      }
      return IGF.Dominance->dominates(dominatingPoint.as<PILBasicBlock>(),
                                      activePoint.as<PILBasicBlock>());
   });

   if (IGM.DebugInfo)
      IGM.DebugInfo->emitFunction(*CurPILFn, CurFn);

   // Map the entry bb.
   LoweredBBs[&*CurPILFn->begin()] = LoweredBB(&CurFn->back(), {});
   // Create LLVM basic blocks for the other bbs.
   for (auto bi = std::next(CurPILFn->begin()), be = CurPILFn->end(); bi != be;
        ++bi) {
      // FIXME: Use the PIL basic block's name.
      llvm::BasicBlock *llBB = llvm::BasicBlock::Create(IGM.getLLVMContext());
      auto phis = emitPHINodesForBBArgs(*this, &*bi, llBB);
      CurFn->getBasicBlockList().push_back(llBB);
      LoweredBBs[&*bi] = LoweredBB(llBB, std::move(phis));
   }

   auto entry = LoweredBBs.begin();
   Builder.SetInsertPoint(entry->second.bb);

   // Map the LLVM arguments to arguments on the entry point BB.
   Explosion params = collectParameters();
   auto funcTy = CurPILFn->getLoweredFunctionType();

   switch (funcTy->getLanguage()) {
      case PILFunctionLanguage::Swift:
         emitEntryPointArgumentsNativeCC(*this, entry->first, params);
         break;
      case PILFunctionLanguage::C:
         emitEntryPointArgumentsCOrObjC(*this, entry->first, params, funcTy);
         break;
   }
   emitLocalSelfMetadata(*this);

   assert(params.empty() && "did not map all llvm params to PIL params?!");

   // It's really nice to be able to assume that we've already emitted
   // all the values from dominating blocks --- it makes simple
   // peepholing more powerful and allows us to avoid the need for
   // nasty "forward-declared" values.  We can do this by emitting
   // blocks using a simple walk through the successor graph.
   //
   // We do want to preserve the original source order, but that's done
   // by having previously added all the primary blocks to the LLVM
   // function in their original order.  As long as any secondary
   // blocks are inserted after the current IP instead of at the end
   // of the function, we're fine.

   // Invariant: for every block in the work queue, we have visited all
   // of its dominators.
   llvm::SmallPtrSet<PILBasicBlock*, 8> visitedBlocks;
   SmallVector<PILBasicBlock*, 8> workQueue; // really a stack

   // Queue up the entry block, for which the invariant trivially holds.
   visitedBlocks.insert(&*CurPILFn->begin());
   workQueue.push_back(&*CurPILFn->begin());

   while (!workQueue.empty()) {
      auto bb = workQueue.pop_back_val();

      // Emit the block.
      visitPILBasicBlock(bb);

#ifndef NDEBUG
      // Assert that the current IR IP (if valid) is immediately prior
      // to the initial IR block for the next primary PIL block.
      // It's not semantically necessary to preserve PIL block order,
      // but we really should.
      if (auto curBB = Builder.GetInsertBlock()) {
         auto next = std::next(PILFunction::iterator(bb));
         if (next != CurPILFn->end()) {
            auto nextBB = LoweredBBs[&*next].bb;
            assert(&*std::next(curBB->getIterator()) == nextBB &&
                   "lost source PIL order?");
         }
      }
#endif

      // The immediate dominator of a successor of this block needn't be
      // this block, but it has to be something which dominates this
      // block.  In either case, we've visited it.
      //
      // Therefore the invariant holds of all the successors, and we can
      // queue them up if we haven't already visited them.
      for (auto *succBB : bb->getSuccessorBlocks()) {
         if (visitedBlocks.insert(succBB).second)
            workQueue.push_back(succBB);
      }
   }

   // If there are dead blocks in the PIL function, we might have left
   // invalid blocks in the IR.  Do another pass and kill them off.
   for (PILBasicBlock &bb : *CurPILFn)
      if (!visitedBlocks.count(&bb))
         LoweredBBs[&bb].bb->eraseFromParent();

}

void IRGenPILFunction::estimateStackSize() {
   if (EstimatedStackSize >= 0)
      return;

   // TODO: as soon as we generate alloca instructions with accurate lifetimes
   // we should also do a better stack size calculation here. Currently we
   // add all stack sizes even if life ranges do not overlap.
   for (PILBasicBlock &BB : *CurPILFn) {
      for (PILInstruction &I : BB) {
         if (auto *ASI = dyn_cast<AllocStackInst>(&I)) {
            const TypeInfo &type = getTypeInfo(ASI->getElementType());
            if (llvm::Constant *SizeConst = type.getStaticSize(IGM)) {
               auto *SizeInt = cast<llvm::ConstantInt>(SizeConst);
               EstimatedStackSize += (int)SizeInt->getSExtValue();
            }
         }
      }
   }
}

void IRGenPILFunction::visitPILBasicBlock(PILBasicBlock *BB) {
   // Insert into the lowered basic block.
   llvm::BasicBlock *llBB = getLoweredBB(BB).bb;
   Builder.SetInsertPoint(llBB);

   bool InEntryBlock = BB->pred_empty();

   // Set this block as the dominance point.  This implicitly communicates
   // with the dominance resolver configured in emitPILFunction.
   DominanceScope dominance(*this, InEntryBlock ? DominancePoint::universal()
                                                : DominancePoint(BB));

   // Generate the body.
   bool InCleanupBlock = false;
   bool KeepCurrentLocation = false;

   for (auto &I : *BB) {
      if (IGM.DebugInfo) {
         // Set the debug info location for I, if applicable.
         auto DS = I.getDebugScope();
         PILLocation ILoc = I.getLoc();
         // Handle cleanup locations.
         if (ILoc.is<CleanupLocation>()) {
            // Cleanup locations point to the decl of the value that is
            // being destroyed (for diagnostic generation). As far as
            // the linetable is concerned, cleanups at the end of a
            // lexical scope should point to the cleanup location, which
            // is the location of the last instruction in the basic block.
            if (!InCleanupBlock) {
               InCleanupBlock = true;
               // Scan ahead to see if this is the final cleanup block in
               // this basic block.
               auto It = I.getIterator();
               do ++It; while (It != BB->end() &&
                               It->getLoc().is<CleanupLocation>());
               // We are still in the middle of a basic block?
               if (It != BB->end() && !isa<TermInst>(It))
                  KeepCurrentLocation = true;
            }

            // Assign the cleanup location to this instruction.
            if (!KeepCurrentLocation) {
               assert(BB->getTerminator());
               ILoc = BB->getTerminator()->getLoc();
               DS = BB->getTerminator()->getDebugScope();
            }
         } else if (InCleanupBlock) {
            KeepCurrentLocation = false;
            InCleanupBlock = false;
         }

         // Until PILDebugScopes are properly serialized, bare functions
         // are allowed to not have a scope.
         if (!DS) {
            if (CurPILFn->isBare())
               DS = CurPILFn->getDebugScope();
            assert(maybeScopeless(I) && "instruction has location, but no scope");
         }

         // Set the builder's debug location.
         if (DS && !KeepCurrentLocation)
            IGM.DebugInfo->setCurrentLoc(Builder, DS, ILoc);
         else {
            // Reuse the last scope for an easier-to-read line table.
            auto Prev = --I.getIterator();
            if (Prev != BB->end())
               DS = Prev->getDebugScope();

            // Use an artificial (line 0) location, to indicate we'd like to
            // reuse the last debug loc.
            IGM.DebugInfo->setCurrentLoc(
               Builder, DS, RegularLocation::getAutoGeneratedLocation());
         }

         if (isa<TermInst>(&I))
            emitDebugVariableRangeExtension(BB);
      }
      visit(&I);
   }

   assert(Builder.hasPostTerminatorIP() && "PIL bb did not terminate block?!");
}

void IRGenPILFunction::visitFunctionRefBaseInst(FunctionRefBaseInst *i) {
   auto fn = i->getInitiallyReferencedFunction();

   llvm::Constant *fnPtr = IGM.getAddrOfPILFunction(
      fn, NotForDefinition, false /*isDynamicallyReplaceableImplementation*/,
      isa<PreviousDynamicFunctionRefInst>(i));

   auto sig = IGM.getSignature(fn->getLoweredFunctionType());

   // Note that the pointer value returned by getAddrOfPILFunction doesn't
   // necessarily have element type sig.getType(), e.g. if it's imported.

   FunctionPointer fp = FunctionPointer::forDirect(fnPtr, sig);

   // Store the function as a FunctionPointer so we can avoid bitcasting
   // or thunking if we don't need to.
   setLoweredFunctionPointer(i, fp);
}

void IRGenPILFunction::visitFunctionRefInst(FunctionRefInst *i) {
   visitFunctionRefBaseInst(i);
}

void IRGenPILFunction::visitDynamicFunctionRefInst(DynamicFunctionRefInst *i) {
   visitFunctionRefBaseInst(i);
}

void IRGenPILFunction::visitPreviousDynamicFunctionRefInst(
   PreviousDynamicFunctionRefInst *i) {
   if (UseBasicDynamicReplacement) {
      IGM.unimplemented(i->getLoc().getSourceLoc(),
                        ": calling the original implementation of a dynamic function is not "
                        "supported with -Xllvm -basic-dynamic-replacement");
   }
   visitFunctionRefBaseInst(i);
}

void IRGenPILFunction::visitAllocGlobalInst(AllocGlobalInst *i) {
   PILGlobalVariable *var = i->getReferencedGlobal();
   PILType loweredTy = var->getLoweredType();
   auto &ti = getTypeInfo(loweredTy);

   auto expansion = IGM.getResilienceExpansionForLayout(var);

   // If the global is fixed-size in all resilience domains that can see it,
   // we allocated storage for it statically, and there's nothing to do.
   if (ti.isFixedSize(expansion))
      return;

   // Otherwise, the static storage for the global consists of a fixed-size
   // buffer.
   Address addr = IGM.getAddrOfPILGlobalVariable(var, ti,
                                                 NotForDefinition);
   emitAllocateValueInBuffer(*this, loweredTy, addr);
}

void IRGenPILFunction::visitGlobalAddrInst(GlobalAddrInst *i) {
   PILGlobalVariable *var = i->getReferencedGlobal();
   PILType loweredTy = var->getLoweredTypeInContext(getExpansionContext());
   assert(loweredTy == i->getType().getObjectType());
   auto &ti = getTypeInfo(loweredTy);

   auto expansion = IGM.getResilienceExpansionForLayout(var);

   // If the variable is empty in all resilience domains that can see it,
   // don't actually emit a symbol for the global at all, just return undef.
   if (ti.isKnownEmpty(expansion)) {
      setLoweredAddress(i, ti.getUndefAddress());
      return;
   }

   Address addr = IGM.getAddrOfPILGlobalVariable(var, ti,
                                                 NotForDefinition);

   // If the global is fixed-size in all resilience domains that can see it,
   // we allocated storage for it statically, and there's nothing to do.
   if (ti.isFixedSize(expansion)) {
      setLoweredAddress(i, addr);
      return;
   }

   // Otherwise, the static storage for the global consists of a fixed-size
   // buffer; project it.
   addr = emitProjectValueInBuffer(*this, loweredTy, addr);

   setLoweredAddress(i, addr);
}

void IRGenPILFunction::visitGlobalValueInst(GlobalValueInst *i) {
   PILGlobalVariable *var = i->getReferencedGlobal();
   assert(var->isInitializedObject() &&
          "global_value only supported for statically initialized objects");
   PILType loweredTy = var->getLoweredType();
   assert(loweredTy == i->getType());
   auto &ti = getTypeInfo(loweredTy);
   assert(ti.isFixedSize(IGM.getResilienceExpansionForLayout(var)));

   llvm::Value *Ref = IGM.getAddrOfPILGlobalVariable(var, ti,
                                                     NotForDefinition).getAddress();

   auto ClassType = loweredTy.getAstType();
   llvm::Value *Metadata =
      emitClassHeapMetadataRef(*this, ClassType, MetadataValueType::TypeMetadata,
                               MetadataState::Complete);
   llvm::Value *CastAddr = Builder.CreateBitCast(Ref, IGM.RefCountedPtrTy);
   llvm::Value *InitRef = emitInitStaticObjectCall(Metadata, CastAddr, "staticref");
   InitRef = Builder.CreateBitCast(InitRef, Ref->getType());

   Explosion e;
   e.add(InitRef);
   setLoweredExplosion(i, e);
}

void IRGenPILFunction::visitMetatypeInst(polar::MetatypeInst *i) {
   auto metaTy = i->getType().castTo<MetatypeType>();
   Explosion e;
   emitMetatypeRef(*this, metaTy, e);
   setLoweredExplosion(i, e);
}

static llvm::Value *getClassBaseValue(IRGenPILFunction &IGF,
                                      PILValue v) {
   if (v->getType().isAddress()) {
      auto addr = IGF.getLoweredAddress(v);
      return IGF.Builder.CreateLoad(addr);
   }

   Explosion e = IGF.getLoweredExplosion(v);
   return e.claimNext();
}

void IRGenPILFunction::visitValueMetatypeInst(polar::ValueMetatypeInst *i) {
   PILType instanceTy = i->getOperand()->getType();
   auto metaTy = i->getType().castTo<MetatypeType>();

   if (metaTy->getRepresentation() == MetatypeRepresentation::Thin) {
      Explosion empty;
      setLoweredExplosion(i, empty);
      return;
   }

   Explosion e;

   if (instanceTy.getClassOrBoundGenericClass()) {
      e.add(emitDynamicTypeOfHeapObject(*this,
                                        getClassBaseValue(*this, i->getOperand()),
                                        metaTy->getRepresentation(), instanceTy));
   } else if (auto arch = instanceTy.getAs<ArchetypeType>()) {
      if (arch->requiresClass()) {
         e.add(emitDynamicTypeOfHeapObject(*this,
                                           getClassBaseValue(*this, i->getOperand()),
                                           metaTy->getRepresentation(), instanceTy));
      } else {
         Address base = getLoweredAddress(i->getOperand());
         e.add(emitDynamicTypeOfOpaqueArchetype(*this, base,
                                                i->getOperand()->getType()));
         // FIXME: We need to convert this back to an ObjC class for an
         // ObjC metatype representation.
         if (metaTy->getRepresentation() == MetatypeRepresentation::ObjC)
            unimplemented(i->getLoc().getSourceLoc(),
                          "objc metatype of non-class-bounded archetype");
      }
   } else {
      emitMetatypeRef(*this, metaTy, e);
   }

   setLoweredExplosion(i, e);
}

void IRGenPILFunction::visitExistentialMetatypeInst(
   polar::ExistentialMetatypeInst *i) {
   Explosion result;
   PILValue op = i->getOperand();
   PILType opType = op->getType();

   switch (opType.getPreferredExistentialRepresentation()) {
      case ExistentialRepresentation::Metatype: {
         Explosion existential = getLoweredExplosion(op);
         emitMetatypeOfMetatype(*this, existential, opType, result);
         break;
      }
      case ExistentialRepresentation::Class: {
         Explosion existential = getLoweredExplosion(op);
         emitMetatypeOfClassExistential(*this, existential, i->getType(),
                                        opType, result);
         break;
      }
      case ExistentialRepresentation::Boxed: {
         Explosion existential = getLoweredExplosion(op);
         emitMetatypeOfBoxedExistential(*this, existential, opType, result);
         break;
      }
      case ExistentialRepresentation::Opaque: {
         Address existential = getLoweredAddress(op);
         emitMetatypeOfOpaqueExistential(*this, existential, opType, result);
         break;
      }
      case ExistentialRepresentation::None:
         llvm_unreachable("Bad existential representation");
   }

   setLoweredExplosion(i, result);
}

static void emitApplyArgument(IRGenPILFunction &IGF,
                              PILValue arg,
                              PILType paramType,
                              Explosion &out) {
   bool isSubstituted = (arg->getType() != paramType);

   // For indirect arguments, we just need to pass a pointer.
   if (paramType.isAddress()) {
      // This address is of the substituted type.
      auto addr = IGF.getLoweredAddress(arg);

      // If a substitution is in play, just bitcast the address.
      if (isSubstituted) {
         auto origType = IGF.IGM.getStoragePointerType(paramType);
         addr = IGF.Builder.CreateBitCast(addr, origType);
      }

      out.add(addr.getAddress());
      return;
   }

   // Otherwise, it's an explosion, which we may need to translate,
   // both in terms of explosion level and substitution levels.
   assert(arg->getType().isObject());

   // Fast path: avoid an unnecessary temporary explosion.
   if (!isSubstituted) {
      IGF.getLoweredExplosion(arg, out);
      return;
   }

   Explosion temp = IGF.getLoweredExplosion(arg);
   reemitAsUnsubstituted(IGF, paramType, arg->getType(),
                         temp, out);
}

static llvm::Value *getObjCClassForValue(IRGenFunction &IGF,
                                         llvm::Value *selfValue,
                                         CanAnyMetatypeType selfType) {
   // If we have a Swift metatype, map it to the heap metadata, which
   // will be the Class for an ObjC type.
   switch (selfType->getRepresentation()) {
      case polar::MetatypeRepresentation::ObjC:
         return selfValue;

      case polar::MetatypeRepresentation::Thick:
         // Convert thick metatype to Objective-C metatype.
         return emitClassHeapMetadataRefForMetatype(IGF, selfValue,
                                                    selfType.getInstanceType());

      case polar::MetatypeRepresentation::Thin:
         llvm_unreachable("Cannot convert Thin metatype to ObjC metatype");
   }
   llvm_unreachable("bad metatype representation");
}

static llvm::Value *
emitWitnessTableForLoweredCallee(IRGenPILFunction &IGF,
                                 CanPILFunctionType substCalleeType) {
   // This use of getSelfInstanceType() assumes that the instance type is
   // always a meaningful formal type.
   auto substSelfType = substCalleeType->getSelfInstanceType(IGF.IGM.getPILModule());
   auto substConformance =
      substCalleeType->getWitnessMethodConformanceOrInvalid();

   llvm::Value *argMetadata = IGF.emitTypeMetadataRef(substSelfType);
   llvm::Value *wtable =
      emitWitnessTableRef(IGF, substSelfType, &argMetadata, substConformance);

   return wtable;
}

Callee LoweredValue::getCallee(IRGenFunction &IGF,
                               llvm::Value *selfValue,
                               CalleeInfo &&calleeInfo) const {
   switch (kind) {
      case Kind::FunctionPointer: {
         auto &fn = getFunctionPointer();
         return Callee(std::move(calleeInfo), fn, selfValue);
      }
         // @todo
//      case Kind::ObjCMethod: {
//         const auto &objcMethod = getObjCMethod();
//         assert(selfValue);
//
//         // Convert a metatype 'self' argument to the ObjC class pointer.
//         // FIXME: why on earth is this not correctly represented in PIL?
//         if (auto metatype = dyn_cast<AnyMetatypeType>(
//            calleeInfo.OrigFnType->getSelfParameter()
//               .getArgumentType(IGF.IGM.getPILModule(), calleeInfo.OrigFnType))) {
//            selfValue = getObjCClassForValue(IGF, selfValue, metatype);
//         }
//
//         return getObjCMethodCallee(IGF, objcMethod, selfValue,
//                                    std::move(calleeInfo));
//      }

      case Kind::SingletonExplosion: {
         auto functionValue = getKnownSingletonExplosion();

         switch (calleeInfo.OrigFnType->getRepresentation()) {
            case PILFunctionType::Representation::Block:
               assert(!selfValue && "block function with self?");
               return getBlockPointerCallee(IGF, functionValue, std::move(calleeInfo));

            case PILFunctionType::Representation::ObjCMethod:
            case PILFunctionType::Representation::Thick:
               llvm_unreachable("unexpected function with singleton representation");

            case PILFunctionType::Representation::WitnessMethod:
            case PILFunctionType::Representation::Thin:
            case PILFunctionType::Representation::Closure:
            case PILFunctionType::Representation::Method:
               return getPolarphpFunctionPointerCallee(IGF, functionValue, selfValue,
                                                       std::move(calleeInfo), false);

            case PILFunctionType::Representation::CFunctionPointer:
               assert(!selfValue && "C function pointer has self?");
               return getCFunctionPointerCallee(IGF, functionValue,
                                                std::move(calleeInfo));
         }
         llvm_unreachable("bad kind");
      }

      case Kind::ExplosionVector: {
         auto vector = getKnownExplosionVector();
         assert(calleeInfo.OrigFnType->getRepresentation()
                == PILFunctionType::Representation::Thick);

         assert(!selfValue && "thick function pointer with self?");
         assert(vector.size() == 2 && "thick function pointer with size != 2");
         llvm::Value *functionValue = vector[0];
         llvm::Value *contextValue = vector[1];
         bool castToRefcountedContext = calleeInfo.OrigFnType->isNoEscape();
         return getPolarphpFunctionPointerCallee(IGF, functionValue, contextValue,
                                                 std::move(calleeInfo),
                                                 castToRefcountedContext);
      }

      case LoweredValue::Kind::EmptyExplosion:
      case LoweredValue::Kind::OwnedAddress:
      case LoweredValue::Kind::ContainedAddress:
      case LoweredValue::Kind::StackAddress:
      case LoweredValue::Kind::DynamicallyEnforcedAddress:
      case LoweredValue::Kind::CoroutineState:
         llvm_unreachable("not a valid callee");
   }
   llvm_unreachable("bad kind");
}

static CallEmission getCallEmissionForLoweredValue(IRGenPILFunction &IGF,
                                                   CanPILFunctionType origCalleeType,
                                                   CanPILFunctionType substCalleeType,
                                                   const LoweredValue &lv,
                                                   llvm::Value *selfValue,
                                                   SubstitutionMap substitutions,
                                                   WitnessMetadata *witnessMetadata,
                                                   Explosion &args) {
   Callee callee = lv.getCallee(IGF, selfValue,
                                CalleeInfo(origCalleeType, substCalleeType,
                                           substitutions));

   switch (origCalleeType->getRepresentation()) {
      case PILFunctionType::Representation::WitnessMethod: {
         auto wtable = emitWitnessTableForLoweredCallee(IGF, substCalleeType);
         witnessMetadata->SelfWitnessTable = wtable;
         break;
      }

      case PILFunctionType::Representation::ObjCMethod:
      case PILFunctionType::Representation::Thick:
      case PILFunctionType::Representation::Block:
      case PILFunctionType::Representation::Thin:
      case PILFunctionType::Representation::CFunctionPointer:
      case PILFunctionType::Representation::Method:
      case PILFunctionType::Representation::Closure:
         break;
   }

   CallEmission callEmission(IGF, std::move(callee));
   if (IGF.CurPILFn->isThunk())
      callEmission.addAttribute(llvm::AttributeList::FunctionIndex,
                                llvm::Attribute::NoInline);

   return callEmission;
}

void IRGenPILFunction::visitBuiltinInst(polar::BuiltinInst *i) {
   const BuiltinInfo &builtin = getPILModule().getBuiltinInfo(i->getName());

   auto argValues = i->getArguments();
   Explosion args;

   for (auto idx : indices(argValues)) {
      auto argValue = argValues[idx];

      // Builtin arguments should never be substituted, so use the value's type
      // as the parameter type.
      emitApplyArgument(*this, argValue, argValue->getType(), args);
   }

   Explosion result;
   emitBuiltinCall(*this, builtin, i->getName(), i->getType(),
                   args, result, i->getSubstitutions());

   setLoweredExplosion(i, result);
}

void IRGenPILFunction::visitApplyInst(polar::ApplyInst *i) {
   visitFullApplySite(i);
}

void IRGenPILFunction::visitTryApplyInst(polar::TryApplyInst *i) {
   visitFullApplySite(i);
}

void IRGenPILFunction::visitFullApplySite(FullApplySite site) {
   const LoweredValue &calleeLV = getLoweredValue(site.getCallee());

   auto origCalleeType = site.getOrigCalleeType();
   auto substCalleeType = site.getSubstCalleeType();

   auto args = site.getArguments();
   PILFunctionConventions origConv(origCalleeType, getPILModule());
   assert(origConv.getNumPILArguments() == args.size());

   // Extract 'self' if it needs to be passed as the context parameter.
   llvm::Value *selfValue = nullptr;
   if (hasSelfContextParameter(origCalleeType)) {
      PILValue selfArg = args.back();
      args = args.drop_back();

      if (selfArg->getType().isObject()) {
         selfValue = getLoweredSingletonExplosion(selfArg);
      } else {
         selfValue = getLoweredAddress(selfArg).getAddress();
      }
   }

   Explosion llArgs;
   WitnessMetadata witnessMetadata;
   CallEmission emission =
      getCallEmissionForLoweredValue(*this, origCalleeType, substCalleeType,
                                     calleeLV, selfValue,
                                     site.getSubstitutionMap(),
                                     &witnessMetadata, llArgs);

   // Lower the arguments and return value in the callee's generic context.
   GenericContextScope scope(IGM, origCalleeType->getInvocationGenericSignature());

   // Allocate space for the coroutine buffer.
   Optional<Address> coroutineBuffer;
   switch (origCalleeType->getCoroutineKind()) {
      case PILCoroutineKind::None:
         break;

      case PILCoroutineKind::YieldOnce:
         coroutineBuffer = emitAllocYieldOnceCoroutineBuffer(*this);
         break;

      case PILCoroutineKind::YieldMany:
         coroutineBuffer = emitAllocYieldManyCoroutineBuffer(*this);
         break;
   }
   if (coroutineBuffer) {
      llArgs.add(coroutineBuffer->getAddress());
   }

   // Lower the PIL arguments to IR arguments.

   // Turn the formal PIL parameters into IR-gen things.
   for (auto index : indices(args)) {
      emitApplyArgument(*this, args[index], origConv.getPILArgumentType(index),
                        llArgs);
   }

   // Pass the generic arguments.
   if (hasPolymorphicParameters(origCalleeType)) {
      SubstitutionMap subMap = site.getSubstitutionMap();
      emitPolymorphicArguments(*this, origCalleeType,
                               subMap, &witnessMetadata, llArgs);
   }

   // Add all those arguments.
   emission.setArgs(llArgs, false, &witnessMetadata);

   PILInstruction *i = site.getInstruction();

   Explosion result;
   emission.emitToExplosion(result, false);

   // For a simple apply, just bind the apply result to the result of the call.
   if (auto apply = dyn_cast<ApplyInst>(i)) {
      setLoweredExplosion(apply, result);

      // For begin_apply, we have to destructure the call.
   } else if (auto beginApply = dyn_cast<BeginApplyInst>(i)) {
      // Grab the continuation pointer.  This will still be an i8*.
      auto continuation = result.claimNext();

      setLoweredCoroutine(beginApply->getTokenResult(),
                          { *coroutineBuffer,
                            continuation,
                            emission.claimTemporaries() });

      setCorrespondingLoweredValues(beginApply->getYieldedValues(), result);

   } else {
      auto tryApplyInst = cast<TryApplyInst>(i);

      // Load the error value.
      PILFunctionConventions substConv(substCalleeType, getPILModule());
      PILType errorType = substConv.getPILErrorType();
      Address errorSlot = getErrorResultSlot(errorType);
      auto errorValue = Builder.CreateLoad(errorSlot);

      auto &normalDest = getLoweredBB(tryApplyInst->getNormalBB());
      auto &errorDest = getLoweredBB(tryApplyInst->getErrorBB());

      // Zero the error slot to maintain the invariant that it always
      // contains null.  This will frequently become a dead store.
      auto nullError = llvm::Constant::getNullValue(errorValue->getType());
      if (!tryApplyInst->getErrorBB()->getSinglePredecessorBlock()) {
         // Only do that here if we can't move the store to the error block.
         // See below.
         Builder.CreateStore(nullError, errorSlot);
      }

      // If the error value is non-null, branch to the error destination.
      auto hasError = Builder.CreateICmpNE(errorValue, nullError);
      Builder.CreateCondBr(hasError, errorDest.bb, normalDest.bb);

      // Set up the PHI nodes on the normal edge.
      unsigned firstIndex = 0;
      addIncomingExplosionToPHINodes(*this, normalDest, firstIndex, result);
      assert(firstIndex == normalDest.phis.size());

      // Set up the PHI nodes on the error edge.
      assert(errorDest.phis.size() == 1);
      errorDest.phis[0]->addIncoming(errorValue, Builder.GetInsertBlock());

      if (tryApplyInst->getErrorBB()->getSinglePredecessorBlock()) {
         // Zeroing out the error slot only in the error block increases the chance
         // that it will become a dead store.
         auto origBB = Builder.GetInsertBlock();
         Builder.SetInsertPoint(errorDest.bb);
         Builder.CreateStore(nullError, errorSlot);
         Builder.SetInsertPoint(origBB);
      }
   }
}

/// If the value is a @convention(witness_method) function, the context
/// is the witness table that must be passed to the call.
///
/// \param v A value of possibly-polymorphic PILFunctionType.
/// \param subs This is the set of substitutions that we are going to be
/// applying to 'v'.
static std::tuple<FunctionPointer, llvm::Value*, CanPILFunctionType>
getPartialApplicationFunction(IRGenPILFunction &IGF, PILValue v,
                              SubstitutionMap subs,
                              CanPILFunctionType substFnType) {
   LoweredValue &lv = IGF.getLoweredValue(v);
   auto fnType = v->getType().castTo<PILFunctionType>();

   switch (lv.kind) {
      case LoweredValue::Kind::ContainedAddress:
      case LoweredValue::Kind::StackAddress:
      case LoweredValue::Kind::DynamicallyEnforcedAddress:
      case LoweredValue::Kind::OwnedAddress:
      case LoweredValue::Kind::EmptyExplosion:
      case LoweredValue::Kind::CoroutineState:
         llvm_unreachable("not a valid function");

      case LoweredValue::Kind::ObjCMethod:
         llvm_unreachable("objc method partial application shouldn't get here");

      case LoweredValue::Kind::FunctionPointer: {
         llvm::Value *context = nullptr;
         switch (fnType->getRepresentation()) {
            case PILFunctionTypeRepresentation::CFunctionPointer:
            case PILFunctionTypeRepresentation::Block:
            case PILFunctionTypeRepresentation::ObjCMethod:
               llvm_unreachable("partial_apply of foreign functions not implemented");

            case PILFunctionTypeRepresentation::WitnessMethod:
               context = emitWitnessTableForLoweredCallee(IGF, substFnType);
               break;
            case PILFunctionTypeRepresentation::Thick:
            case PILFunctionTypeRepresentation::Thin:
            case PILFunctionTypeRepresentation::Method:
            case PILFunctionTypeRepresentation::Closure:
               break;
         }

         auto fn = lv.getFunctionPointer();
         return std::make_tuple(fn, context, fnType);
      }
      case LoweredValue::Kind::SingletonExplosion: {
         llvm::Value *fnPtr = lv.getKnownSingletonExplosion();
         auto fn = FunctionPointer::forExplosionValue(IGF, fnPtr, fnType);
         llvm::Value *context = nullptr;
         auto repr = fnType->getRepresentation();
         assert(repr != PILFunctionType::Representation::Block &&
                "partial apply of block not implemented");
         if (repr == PILFunctionType::Representation::WitnessMethod) {
            context = emitWitnessTableForLoweredCallee(IGF, substFnType);
         }
         return std::make_tuple(fn, context, fnType);
      }
      case LoweredValue::Kind::ExplosionVector: {
         assert(fnType->getRepresentation()
                == PILFunctionType::Representation::Thick);
         Explosion ex = lv.getExplosion(IGF, v->getType());
         llvm::Value *fnPtr = ex.claimNext();
         auto fn = FunctionPointer::forExplosionValue(IGF, fnPtr, fnType);
         llvm::Value *context = ex.claimNext();
         return std::make_tuple(fn, context, fnType);
      }
   }
   llvm_unreachable("bad kind");
}

void IRGenPILFunction::visitPartialApplyInst(polar::PartialApplyInst *i) {
   PILValue v(i);

   // NB: We collect the arguments under the substituted type.
   auto args = i->getArguments();
   auto calleeTy = i->getSubstCalleeType();
   auto params = calleeTy->getParameters();
   params = params.slice(params.size() - args.size(), args.size());

   Explosion llArgs;

   // Lower the parameters in the callee's generic context.
   {
      GenericContextScope scope(IGM,
                                i->getOrigCalleeType()->getSubstGenericSignature());
      for (auto index : indices(args)) {
         auto paramTy = IGM.silConv.getPILType(params[index], calleeTy);
         assert(args[index]->getType() == paramTy);
         emitApplyArgument(*this, args[index], paramTy, llArgs);
      }
   }

   auto &lv = getLoweredValue(i->getCallee());
   // @todo
//   if (lv.kind == LoweredValue::Kind::ObjCMethod) {
//      // Objective-C partial applications require a different path. There's no
//      // actual function pointer to capture, and we semantically can't cache
//      // dispatch, so we need to perform the message send in the partial
//      // application thunk.
//      auto &objcMethod = lv.getObjCMethod();
//      assert(i->getArguments().size() == 1 &&
//             "only partial application of objc method to self implemented");
//      assert(llArgs.size() == 1 &&
//             "objc partial_apply argument is not a single retainable pointer?!");
//      llvm::Value *selfVal = llArgs.claimNext();
//
//      Explosion function;
//      emitObjCPartialApplication(*this,
//                                 objcMethod,
//                                 i->getOrigCalleeType(),
//                                 i->getType().castTo<PILFunctionType>(),
//                                 selfVal,
//                                 i->getArguments()[0]->getType(),
//                                 function);
//      setLoweredExplosion(i, function);
//      return;
//   }

   // Get the function value.
   auto result = getPartialApplicationFunction(*this, i->getCallee(),
                                               i->getSubstitutionMap(),
                                               i->getSubstCalleeType());
   FunctionPointer calleeFn = std::get<0>(result);
   llvm::Value *innerContext = std::get<1>(result);
   CanPILFunctionType origCalleeTy = std::get<2>(result);

   // Create the thunk and function value.
   Explosion function;
   auto closureStackAddr = emitFunctionPartialApplication(
      *this, *CurPILFn, calleeFn, innerContext, llArgs, params,
      i->getSubstitutionMap(), origCalleeTy, i->getSubstCalleeType(),
      i->getType().castTo<PILFunctionType>(), function, false);
   setLoweredExplosion(v, function);

   if (closureStackAddr) {
      assert(i->isOnStack());
      LoweredPartialApplyAllocations[v] = *closureStackAddr;
   }
}

void IRGenPILFunction::visitIntegerLiteralInst(polar::IntegerLiteralInst *i) {
   Explosion e;
   if (i->getType().is<BuiltinIntegerLiteralType>()) {
      auto pair = emitConstantIntegerLiteral(IGM, i);
      e.add(pair.Data);
      e.add(pair.Flags);
   } else {
      llvm::Value *constant = emitConstantInt(IGM, i);
      e.add(constant);
   }
   setLoweredExplosion(i, e);
}

void IRGenPILFunction::visitFloatLiteralInst(polar::FloatLiteralInst *i) {
   llvm::Value *constant = emitConstantFP(IGM, i);
   Explosion e;
   e.add(constant);
   setLoweredExplosion(i, e);
}

void IRGenPILFunction::visitStringLiteralInst(polar::StringLiteralInst *i) {
   llvm::Value *addr;

   // Emit a load of a selector.
//   if (i->getEncoding() == polar::StringLiteralInst::Encoding::ObjCSelector)
//      addr = emitObjCSelectorRefLoad(i->getValue());
//   else
   /// TODO
   addr = emitAddrOfConstantString(IGM, i);

   Explosion e;
   e.add(addr);
   setLoweredExplosion(i, e);
}

void IRGenPILFunction::visitUnreachableInst(polar::UnreachableInst *i) {
   Builder.CreateUnreachable();
}

static void emitCoroutineExit(IRGenPILFunction &IGF) {
   // The LLVM coroutine representation demands that there be a
   // unique call to llvm.coro.end.

   // If the coroutine exit block already exists, just branch to it.
   if (auto coroEndBB = IGF.CoroutineExitBlock) {
      IGF.Builder.CreateBr(coroEndBB);
      return;
   }

   // Otherwise, create it and branch to it.
   auto coroEndBB = IGF.createBasicBlock("coro.end");
   IGF.CoroutineExitBlock = coroEndBB;
   IGF.Builder.CreateBr(coroEndBB);

   // Emit the block.
   IGF.Builder.emitBlock(coroEndBB);
   auto handle = IGF.getCoroutineHandle();
   IGF.Builder.CreateIntrinsicCall(llvm::Intrinsic::ID::coro_end, {
      handle,
      /*is unwind*/ IGF.Builder.getFalse()
   });
   IGF.Builder.CreateUnreachable();
}

static void emitReturnInst(IRGenPILFunction &IGF,
                           PILType resultTy,
                           Explosion &result) {
   // If we're generating a coroutine, just call coro.end.
   if (IGF.isCoroutine()) {
      assert(result.empty() &&
             "coroutines do not currently support non-void returns");
      emitCoroutineExit(IGF);
      return;
   }

   // The invariant on the out-parameter is that it's always zeroed, so
   // there's nothing to do here.

   // Even if PIL has a direct return, the IR-level calling convention may
   // require an indirect return.
   if (IGF.IndirectReturn.isValid()) {
      auto &retTI = cast<LoadableTypeInfo>(IGF.getTypeInfo(resultTy));
      retTI.initialize(IGF, result, IGF.IndirectReturn, false);
      IGF.Builder.CreateRetVoid();
   } else {
      auto funcLang = IGF.CurPILFn->getLoweredFunctionType()->getLanguage();
      auto swiftCCReturn = funcLang == PILFunctionLanguage::Swift;
      assert(swiftCCReturn ||
             funcLang == PILFunctionLanguage::C && "Need to handle all cases");
      PILFunctionConventions conv(IGF.CurPILFn->getLoweredFunctionType(),
                                  IGF.getPILModule());
      auto funcResultType =
         IGF.CurPILFn->mapTypeIntoContext(conv.getPILResultType());
      IGF.emitScalarReturn(resultTy, funcResultType, result, swiftCCReturn,
                           false);
   }
}

void IRGenPILFunction::visitReturnInst(polar::ReturnInst *i) {
   Explosion result = getLoweredExplosion(i->getOperand());

   // Implicitly autorelease the return value if the function's result
   // convention is autoreleased.
   auto fnConv = CurPILFn->getConventions();
   if (fnConv.getNumDirectPILResults() == 1
       && (fnConv.getDirectPILResults().begin()->getConvention()
           == ResultConvention::Autoreleased)) {
      Explosion temp;
      // @todo
//      temp.add(emitObjCAutoreleaseReturnValue(*this, result.claimNext()));
      result = std::move(temp);
   }

   emitReturnInst(*this, i->getOperand()->getType(), result);
}

void IRGenPILFunction::visitThrowInst(polar::ThrowInst *i) {
   // Store the exception to the error slot.
   llvm::Value *exn = getLoweredSingletonExplosion(i->getOperand());

   Builder.CreateStore(exn, getCallerErrorResultSlot());

   // Create a normal return, but leaving the return value undefined.
   auto fnTy = CurFn->getType()->getPointerElementType();
   auto retTy = cast<llvm::FunctionType>(fnTy)->getReturnType();
   if (retTy->isVoidTy()) {
      Builder.CreateRetVoid();
   } else {
      Builder.CreateRet(llvm::UndefValue::get(retTy));
   }
}

void IRGenPILFunction::visitUnwindInst(polar::UnwindInst *i) {
   // Just call coro.end; there's no need to distinguish 'unwind'
   // and 'return' at the LLVM level.
   emitCoroutineExit(*this);
}

void IRGenPILFunction::visitYieldInst(polar::YieldInst *i) {
   auto coroutineType = CurPILFn->getLoweredFunctionType();
   PILFunctionConventions coroConv(coroutineType, getPILModule());

   GenericContextScope scope(IGM, coroutineType->getInvocationGenericSignature());

   // Collect all the yielded values.
   Explosion values;
   auto yieldedValues = i->getYieldedValues();
   auto yields = coroutineType->getYields();
   assert(yieldedValues.size() == yields.size());
   for (auto idx : indices(yieldedValues)) {
      PILValue value = yieldedValues[idx];
      PILParameterInfo yield = yields[idx];
      emitApplyArgument(*this, value, coroConv.getPILType(yield), values);
   }

   // Emit the yield intrinsic.
   auto isUnwind = emitYield(*this, coroutineType, values);

   // Branch to the appropriate destination.
   auto unwindBB = getLoweredBB(i->getUnwindBB()).bb;
   auto resumeBB = getLoweredBB(i->getResumeBB()).bb;
   Builder.CreateCondBr(isUnwind, unwindBB, resumeBB);
}

void IRGenPILFunction::visitBeginApplyInst(BeginApplyInst *i) {
   visitFullApplySite(i);
}

void IRGenPILFunction::visitEndApplyInst(EndApplyInst *i) {
   visitEndApply(i->getBeginApply(), false);
}

void IRGenPILFunction::visitAbortApplyInst(AbortApplyInst *i) {
   visitEndApply(i->getBeginApply(), true);
}

void IRGenPILFunction::visitEndApply(BeginApplyInst *i, bool isAbort) {
   const auto &coroutine = getLoweredCoroutine(i->getTokenResult());

   auto sig = Signature::forCoroutineContinuation(IGM, i->getOrigCalleeType());

   // Cast the continuation pointer to the right function pointer type.
   auto continuation = coroutine.Continuation;
   continuation = Builder.CreateBitCast(continuation,
                                        sig.getType()->getPointerTo());

   FunctionPointer callee(continuation, sig);

   Builder.CreateCall(callee, {
      coroutine.Buffer.getAddress(),
      llvm::ConstantInt::get(IGM.Int1Ty, isAbort)
   });

   coroutine.Temporaries.destroyAll(*this);

   emitDeallocYieldOnceCoroutineBuffer(*this, coroutine.Buffer);
}

static llvm::BasicBlock *emitBBMapForSwitchValue(
   IRGenPILFunction &IGF,
   SmallVectorImpl<std::pair<PILValue, llvm::BasicBlock*>> &dests,
   SwitchValueInst *inst) {
   for (unsigned i = 0, e = inst->getNumCases(); i < e; ++i) {
      auto casePair = inst->getCase(i);
      dests.push_back({casePair.first, IGF.getLoweredBB(casePair.second).bb});
   }

   llvm::BasicBlock *defaultDest = nullptr;
   if (inst->hasDefault())
      defaultDest = IGF.getLoweredBB(inst->getDefaultBB()).bb;
   return defaultDest;
}

static llvm::ConstantInt *
getSwitchCaseValue(IRGenFunction &IGF, PILValue val) {
   auto *IL = cast<IntegerLiteralInst>(val);
   return cast<llvm::ConstantInt>(emitConstantInt(IGF.IGM, IL));
}

static void
emitSwitchValueDispatch(IRGenPILFunction &IGF,
                        PILType ty,
                        Explosion &value,
                        ArrayRef<std::pair<PILValue, llvm::BasicBlock*>> dests,
                        llvm::BasicBlock *defaultDest) {
   // Create an unreachable block for the default if the original PIL
   // instruction had none.
   bool unreachableDefault = false;
   if (!defaultDest) {
      unreachableDefault = true;
      defaultDest = llvm::BasicBlock::Create(IGF.IGM.getLLVMContext());
   }

   if (ty.is<BuiltinIntegerType>()) {
      auto *discriminator = value.claimNext();
      auto *i = IGF.Builder.CreateSwitch(discriminator, defaultDest,
                                         dests.size());
      for (auto &dest : dests)
         i->addCase(getSwitchCaseValue(IGF, dest.first), dest.second);
   } else {
      // Get the value we're testing, which is a function.
      llvm::Value *val;
      llvm::BasicBlock *nextTest = nullptr;
      if (ty.is<PILFunctionType>()) {
         val = value.claimNext();   // Function pointer.
         //values.claimNext();         // Ignore the data pointer.
      } else {
         llvm_unreachable("switch_value operand has an unknown type");
      }

      for (int i = 0, e = dests.size(); i < e; ++i) {
         auto casePair = dests[i];
         llvm::Value *caseval;
         auto casevalue = IGF.getLoweredExplosion(casePair.first);
         if (casePair.first->getType().is<PILFunctionType>()) {
            caseval = casevalue.claimNext();   // Function pointer.
            //values.claimNext();         // Ignore the data pointer.
         } else {
            llvm_unreachable("switch_value operand has an unknown type");
         }

         // Compare operand with a case tag value.
         llvm::Value *cond = IGF.Builder.CreateICmp(llvm::CmpInst::ICMP_EQ,
                                                    val, caseval);

         if (i == e -1 && !unreachableDefault) {
            nextTest = nullptr;
            IGF.Builder.CreateCondBr(cond, casePair.second, defaultDest);
         } else {
            nextTest = IGF.createBasicBlock("next-test");
            IGF.Builder.CreateCondBr(cond, casePair.second, nextTest);
            IGF.Builder.emitBlock(nextTest);
            IGF.Builder.SetInsertPoint(nextTest);
         }
      }

      if (nextTest) {
         IGF.Builder.CreateBr(defaultDest);
      }
   }

   if (unreachableDefault) {
      IGF.Builder.emitBlock(defaultDest);
      IGF.Builder.CreateUnreachable();
   }
}

void IRGenPILFunction::visitSwitchValueInst(SwitchValueInst *inst) {
   Explosion value = getLoweredExplosion(inst->getOperand());

   // Map the PIL dest bbs to their LLVM bbs.
   SmallVector<std::pair<PILValue, llvm::BasicBlock*>, 4> dests;
   auto *defaultDest = emitBBMapForSwitchValue(*this, dests, inst);

   emitSwitchValueDispatch(*this, inst->getOperand()->getType(),
                           value, dests, defaultDest);
}

// Bind an incoming explosion value to an explosion of LLVM phi node(s).
static void addIncomingExplosionToPHINodes(IRGenPILFunction &IGF,
                                           ArrayRef<llvm::Value*> phis,
                                           Explosion &argValue) {
   llvm::BasicBlock *curBB = IGF.Builder.GetInsertBlock();
   unsigned phiIndex = 0;
   while (!argValue.empty())
      cast<llvm::PHINode>(phis[phiIndex++])
         ->addIncoming(argValue.claimNext(), curBB);
   assert(phiIndex == phis.size() && "explosion doesn't match number of phis");
}

// Bind an incoming explosion value to a PILArgument's LLVM phi node(s).
static void addIncomingExplosionToPHINodes(IRGenPILFunction &IGF,
                                           LoweredBB &lbb,
                                           unsigned &phiIndex,
                                           Explosion &argValue) {
   llvm::BasicBlock *curBB = IGF.Builder.GetInsertBlock();
   while (!argValue.empty())
      lbb.phis[phiIndex++]->addIncoming(argValue.claimNext(), curBB);
}

// Bind an incoming address value to a PILArgument's LLVM phi node(s).
static void addIncomingAddressToPHINodes(IRGenPILFunction &IGF,
                                         ArrayRef<llvm::Value*> phis,
                                         Address argValue) {
   llvm::BasicBlock *curBB = IGF.Builder.GetInsertBlock();
   assert(phis.size() == 1 && "more than one phi for address?!");
   cast<llvm::PHINode>(phis[0])->addIncoming(argValue.getAddress(), curBB);
}

// Bind an incoming address value to a PILArgument's LLVM phi node(s).
static void addIncomingAddressToPHINodes(IRGenPILFunction &IGF,
                                         LoweredBB &lbb,
                                         unsigned &phiIndex,
                                         Address argValue) {
   llvm::BasicBlock *curBB = IGF.Builder.GetInsertBlock();
   lbb.phis[phiIndex++]->addIncoming(argValue.getAddress(), curBB);
}

// Add branch arguments to destination phi nodes.
static void addIncomingPILArgumentsToPHINodes(IRGenPILFunction &IGF,
                                              LoweredBB &lbb,
                                              OperandValueArrayRef args) {
   unsigned phiIndex = 0;
   for (PILValue arg : args) {
      if (arg->getType().isAddress()) {
         addIncomingAddressToPHINodes(IGF, lbb, phiIndex,
                                      IGF.getLoweredAddress(arg));
         continue;
      }

      Explosion argValue = IGF.getLoweredExplosion(arg);
      addIncomingExplosionToPHINodes(IGF, lbb, phiIndex, argValue);
   }
}

static llvm::BasicBlock *emitBBMapForSwitchEnum(
   IRGenPILFunction &IGF,
   SmallVectorImpl<std::pair<EnumElementDecl*, llvm::BasicBlock*>> &dests,
   SwitchEnumInstBase *inst) {
   for (unsigned i = 0, e = inst->getNumCases(); i < e; ++i) {
      auto casePair = inst->getCase(i);

      // If the destination BB accepts the case argument, set up a waypoint BB so
      // we can feed the values into the argument's PHI node(s).
      //
      // FIXME: This is cheesy when the destination BB has only the switch
      // as a predecessor.
      if (!casePair.second->args_empty())
         dests.push_back({casePair.first,
                          llvm::BasicBlock::Create(IGF.IGM.getLLVMContext())});
      else
         dests.push_back({casePair.first, IGF.getLoweredBB(casePair.second).bb});
   }

   llvm::BasicBlock *defaultDest = nullptr;
   if (inst->hasDefault())
      defaultDest = IGF.getLoweredBB(inst->getDefaultBB()).bb;
   return defaultDest;
}

void IRGenPILFunction::visitSwitchEnumInst(SwitchEnumInst *inst) {
   Explosion value = getLoweredExplosion(inst->getOperand());

   // Map the PIL dest bbs to their LLVM bbs.
   SmallVector<std::pair<EnumElementDecl*, llvm::BasicBlock*>, 4> dests;
   llvm::BasicBlock *defaultDest
      = emitBBMapForSwitchEnum(*this, dests, inst);

   // Emit the dispatch.
   auto &EIS = getEnumImplStrategy(IGM, inst->getOperand()->getType());
   EIS.emitValueSwitch(*this, value, dests, defaultDest);

   // Bind arguments for cases that want them.
   for (unsigned i = 0, e = inst->getNumCases(); i < e; ++i) {
      auto casePair = inst->getCase(i);

      if (!casePair.second->args_empty()) {
         auto waypointBB = dests[i].second;
         auto &destLBB = getLoweredBB(casePair.second);

         Builder.emitBlock(waypointBB);

         Explosion inValue = getLoweredExplosion(inst->getOperand());
         Explosion projected;
         emitProjectLoadableEnum(*this, inst->getOperand()->getType(),
                                 inValue, casePair.first, projected);

         unsigned phiIndex = 0;
         addIncomingExplosionToPHINodes(*this, destLBB, phiIndex, projected);

         Builder.CreateBr(destLBB.bb);
      }
   }
}

void
IRGenPILFunction::visitSwitchEnumAddrInst(SwitchEnumAddrInst *inst) {
   Address value = getLoweredAddress(inst->getOperand());

   // Map the PIL dest bbs to their LLVM bbs.
   SmallVector<std::pair<EnumElementDecl*, llvm::BasicBlock*>, 4> dests;
   llvm::BasicBlock *defaultDest
      = emitBBMapForSwitchEnum(*this, dests, inst);

   // Emit the dispatch.
   emitSwitchAddressOnlyEnumDispatch(*this, inst->getOperand()->getType(),
                                     value, dests, defaultDest);
}

// FIXME: We could lower select_enum directly to LLVM select in a lot of cases.
// For now, just emit a switch and phi nodes, like a chump.
template <class C, class T, class B>
static llvm::BasicBlock *
emitBBMapForSelect(IRGenPILFunction &IGF, Explosion &resultPHI,
                   SmallVectorImpl<std::pair<T, llvm::BasicBlock *>> &BBs,
                   llvm::BasicBlock *&defaultBB,
                   SelectInstBase<C, T, B> *inst) {

   auto origBB = IGF.Builder.GetInsertBlock();

   // Set up a continuation BB and phi nodes to receive the result value.
   llvm::BasicBlock *contBB = IGF.createBasicBlock("select_enum");
   IGF.Builder.SetInsertPoint(contBB);

   // Emit an explosion of phi node(s) to receive the value.
   SmallVector<llvm::Value*, 4> phis;
   auto &ti = IGF.getTypeInfo(inst->getType());
   emitPHINodesForType(IGF, inst->getType(), ti,
                       inst->getNumCases() + inst->hasDefault(),
                       phis);
   resultPHI.add(phis);

   IGF.Builder.SetInsertPoint(origBB);

   auto addIncoming = [&](PILValue value) {
      if (value->getType().isAddress()) {
         addIncomingAddressToPHINodes(IGF, resultPHI.getAll(),
                                      IGF.getLoweredAddress(value));
      } else {
         Explosion ex = IGF.getLoweredExplosion(value);
         addIncomingExplosionToPHINodes(IGF, resultPHI.getAll(), ex);
      }
   };

   for (unsigned i = 0, e = inst->getNumCases(); i < e; ++i) {
      auto casePair = inst->getCase(i);

      // Create a basic block destination for this case.
      llvm::BasicBlock *destBB = IGF.createBasicBlock("");
      IGF.Builder.emitBlock(destBB);

      // Feed the corresponding result into the phi nodes.
      addIncoming(casePair.second);

      // Jump immediately to the continuation.
      IGF.Builder.CreateBr(contBB);
      BBs.push_back(std::make_pair(casePair.first, destBB));
   }

   if (inst->hasDefault()) {
      defaultBB = IGF.createBasicBlock("");
      IGF.Builder.emitBlock(defaultBB);

      addIncoming(inst->getDefaultResult());

      IGF.Builder.CreateBr(contBB);
   } else {
      defaultBB = nullptr;
   }

   IGF.Builder.emitBlock(contBB);

   IGF.Builder.SetInsertPoint(origBB);
   return contBB;
}

// Try to map the value of a select_enum directly to an int type with a simple
// cast from the tag value to the result type. Optionally also by adding a
// constant offset.
// This is useful, e.g. for rawValue or hashValue of C-like enums.
static llvm::Value *
mapTriviallyToInt(IRGenPILFunction &IGF, const EnumImplStrategy &EIS, SelectEnumInst *inst) {

   // All cases must be covered
   if (inst->hasDefault())
      return nullptr;

   auto &ti = IGF.getTypeInfo(inst->getType());
   ExplosionSchema schema = ti.getSchema();

   // Check if the select_enum's result is a single integer scalar.
   if (schema.size() != 1)
      return nullptr;

   if (!schema[0].isScalar())
      return nullptr;

   llvm::Type *type = schema[0].getScalarType();
   auto *resultType = dyn_cast<llvm::IntegerType>(type);
   if (!resultType)
      return nullptr;

   // Check if the case values directly map to the tag values, maybe with a
   // constant offset.
   APInt commonOffset;
   bool offsetValid = false;

   for (unsigned i = 0, e = inst->getNumCases(); i < e; ++i) {
      auto casePair = inst->getCase(i);

      int64_t index = EIS.getDiscriminatorIndex(casePair.first);
      if (index < 0)
         return nullptr;

      auto *intLit = dyn_cast<IntegerLiteralInst>(casePair.second);
      if (!intLit)
         return nullptr;

      APInt caseValue = intLit->getValue();
      APInt offset = caseValue - index;
      if (offsetValid) {
         if (offset != commonOffset)
            return nullptr;
      } else {
         commonOffset = offset;
         offsetValid = true;
      }
   }

   // Ask the enum implementation strategy to extract the enum tag as an integer
   // value.
   Explosion enumValue = IGF.getLoweredExplosion(inst->getEnumOperand());
   llvm::Value *result = EIS.emitExtractDiscriminator(IGF, enumValue);
   if (!result) {
      (void)enumValue.claimAll();
      return nullptr;
   }

   // Cast to the result type.
   result = IGF.Builder.CreateIntCast(result, resultType, false);
   if (commonOffset != 0) {
      // The offset, if any.
      auto *offsetConst = llvm::ConstantInt::get(resultType, commonOffset);
      result = IGF.Builder.CreateAdd(result, offsetConst);
   }
   return result;
}

template <class C, class T, class B>
static LoweredValue getLoweredValueForSelect(IRGenPILFunction &IGF,
                                             Explosion &result,
                                             SelectInstBase<C, T, B> *inst) {
   if (inst->getType().isAddress())
      // FIXME: Loses potentially better alignment info we might have.
      return LoweredValue(Address(result.claimNext(),
                                  IGF.getTypeInfo(inst->getType()).getBestKnownAlignment()));
   return LoweredValue(result);
}

static void emitSingleEnumMemberSelectResult(IRGenPILFunction &IGF,
                                             SelectEnumInstBase *inst,
                                             llvm::Value *isTrue,
                                             Explosion &result) {
   assert((inst->getNumCases() == 1 && inst->hasDefault()) ||
          (inst->getNumCases() == 2 && !inst->hasDefault()));

   // Extract the true values.
   auto trueValue = inst->getCase(0).second;
   SmallVector<llvm::Value*, 4> TrueValues;
   if (trueValue->getType().isAddress()) {
      TrueValues.push_back(IGF.getLoweredAddress(trueValue).getAddress());
   } else {
      Explosion ex = IGF.getLoweredExplosion(trueValue);
      while (!ex.empty())
         TrueValues.push_back(ex.claimNext());
   }

   // Extract the false values.
   auto falseValue =
      inst->hasDefault() ? inst->getDefaultResult() : inst->getCase(1).second;
   SmallVector<llvm::Value*, 4> FalseValues;
   if (falseValue->getType().isAddress()) {
      FalseValues.push_back(IGF.getLoweredAddress(falseValue).getAddress());
   } else {
      Explosion ex = IGF.getLoweredExplosion(falseValue);
      while (!ex.empty())
         FalseValues.push_back(ex.claimNext());
   }

   assert(TrueValues.size() == FalseValues.size() &&
          "explosions didn't produce same element count?");
   for (unsigned i = 0, e = FalseValues.size(); i != e; ++i) {
      auto *TV = TrueValues[i], *FV = FalseValues[i];
      // It is pretty common to select between zero and 1 as the result of the
      // select.  Instead of emitting an obviously dumb select, emit nothing or
      // a zext.
      if (auto *TC = dyn_cast<llvm::ConstantInt>(TV))
         if (auto *FC = dyn_cast<llvm::ConstantInt>(FV))
            if (TC->isOne() && FC->isZero()) {
               result.add(IGF.Builder.CreateZExtOrBitCast(isTrue, TV->getType()));
               continue;
            }

      result.add(IGF.Builder.CreateSelect(isTrue, TV, FalseValues[i]));
   }
}


void IRGenPILFunction::visitSelectEnumInst(SelectEnumInst *inst) {
   auto &EIS = getEnumImplStrategy(IGM, inst->getEnumOperand()->getType());
   Explosion result;

   if (llvm::Value *R = mapTriviallyToInt(*this, EIS, inst)) {
      result.add(R);
   } else if ((inst->getNumCases() == 1 && inst->hasDefault()) ||
              (inst->getNumCases() == 2 && !inst->hasDefault())) {
      // If this is testing for one case, do simpler codegen.  This is
      // particularly common when testing optionals.
      Explosion value = getLoweredExplosion(inst->getEnumOperand());
      auto isTrue = EIS.emitValueCaseTest(*this, value, inst->getCase(0).first);
      emitSingleEnumMemberSelectResult(*this, inst, isTrue, result);
   } else {
      Explosion value = getLoweredExplosion(inst->getEnumOperand());

      // Map the PIL dest bbs to their LLVM bbs.
      SmallVector<std::pair<EnumElementDecl*, llvm::BasicBlock*>, 4> dests;
      llvm::BasicBlock *defaultDest;
      llvm::BasicBlock *contBB
         = emitBBMapForSelect(*this, result, dests, defaultDest, inst);

      // Emit the dispatch.
      EIS.emitValueSwitch(*this, value, dests, defaultDest);

      // emitBBMapForSelectEnum set up a continuation block and phi nodes to
      // receive the result.
      Builder.SetInsertPoint(contBB);
   }
   setLoweredValue(inst,
                   getLoweredValueForSelect(*this, result, inst));
}

void IRGenPILFunction::visitSelectEnumAddrInst(SelectEnumAddrInst *inst) {
   Address value = getLoweredAddress(inst->getEnumOperand());
   Explosion result;

   if ((inst->getNumCases() == 1 && inst->hasDefault()) ||
       (inst->getNumCases() == 2 && !inst->hasDefault())) {
      auto &EIS = getEnumImplStrategy(IGM, inst->getEnumOperand()->getType());
      // If this is testing for one case, do simpler codegen.  This is
      // particularly common when testing optionals.
      auto isTrue = EIS.emitIndirectCaseTest(*this,
                                             inst->getEnumOperand()->getType(),
                                             value, inst->getCase(0).first);
      emitSingleEnumMemberSelectResult(*this, inst, isTrue, result);
   } else {
      // Map the PIL dest bbs to their LLVM bbs.
      SmallVector<std::pair<EnumElementDecl*, llvm::BasicBlock*>, 4> dests;
      llvm::BasicBlock *defaultDest;
      llvm::BasicBlock *contBB
         = emitBBMapForSelect(*this, result, dests, defaultDest, inst);

      // Emit the dispatch.
      emitSwitchAddressOnlyEnumDispatch(*this, inst->getEnumOperand()->getType(),
                                        value, dests, defaultDest);

      // emitBBMapForSelectEnum set up a phi node to receive the result.
      Builder.SetInsertPoint(contBB);
   }

   setLoweredValue(inst,
                   getLoweredValueForSelect(*this, result, inst));
}

void IRGenPILFunction::visitSelectValueInst(SelectValueInst *inst) {
   Explosion value = getLoweredExplosion(inst->getOperand());

   // Map the PIL dest bbs to their LLVM bbs.
   SmallVector<std::pair<PILValue, llvm::BasicBlock*>, 4> dests;
   llvm::BasicBlock *defaultDest;
   Explosion result;
   auto *contBB = emitBBMapForSelect(*this, result, dests, defaultDest, inst);

   // Emit the dispatch.
   emitSwitchValueDispatch(*this, inst->getOperand()->getType(), value, dests,
                           defaultDest);

   // emitBBMapForSelectEnum set up a continuation block and phi nodes to
   // receive the result.
   Builder.SetInsertPoint(contBB);

   setLoweredValue(inst,
                   getLoweredValueForSelect(*this, result, inst));
}

void IRGenPILFunction::visitDynamicMethodBranchInst(DynamicMethodBranchInst *i){
   /// TODO
//   LoweredBB &hasMethodBB = getLoweredBB(i->getHasMethodBB());
//   LoweredBB &noMethodBB = getLoweredBB(i->getNoMethodBB());
//
//   // Emit the respondsToSelector: call.
//   StringRef selector;
//   llvm::SmallString<64> selectorBuffer;
////   if (auto fnDecl = dyn_cast<FuncDecl>(i->getMember().getDecl()))
////      selector = fnDecl->getObjCSelector().getString(selectorBuffer);
////   else if (auto var = dyn_cast<AbstractStorageDecl>(i->getMember().getDecl()))
////      selector = var->getObjCGetterSelector().getString(selectorBuffer);
////   else
//   // @todo
//   llvm_unreachable("Unhandled dynamic method branch query");
//
//   llvm::Value *object = getLoweredExplosion(i->getOperand()).claimNext();
//   if (object->getType() != IGM.ObjCPtrTy)
//      object = Builder.CreateBitCast(object, IGM.ObjCPtrTy);
////   llvm::Value *loadSel = emitObjCSelectorRefLoad(selector);
////
////   llvm::Value *respondsToSelector
////      = emitObjCSelectorRefLoad("respondsToSelector:");
//
//   llvm::Constant *messenger = IGM.getObjCMsgSendFn();
//   llvm::Type *argTys[] = {
//      IGM.ObjCPtrTy,
//      IGM.Int8PtrTy,
//      IGM.Int8PtrTy,
//   };
//   auto respondsToSelectorTy = llvm::FunctionType::get(IGM.Int1Ty,
//                                                       argTys,
//      /*isVarArg*/ false)
//      ->getPointerTo();
//   messenger = llvm::ConstantExpr::getBitCast(messenger,
//                                              respondsToSelectorTy);
//   llvm::CallInst *call = Builder.CreateCall(messenger,
//                                             {object, respondsToSelector, loadSel});
//   call->setDoesNotThrow();
//
//   // FIXME: Assume (probably safely) that the hasMethodBB has only us as a
//   // predecessor, and cannibalize its bb argument so we can represent is as an
//   // ObjCMethod lowered value. This is hella gross but saves us having to
//   // implement ObjCMethod-to-Explosion lowering and creating a thunk we don't
//   // want.
//   assert(std::next(i->getHasMethodBB()->pred_begin())
//          == i->getHasMethodBB()->pred_end()
//          && "lowering dynamic_method_br with multiple preds for destination "
//             "not implemented");
//   // Kill the existing lowered value for the bb arg and its phi nodes.
//   PILValue methodArg = i->getHasMethodBB()->args_begin()[0];
//   Explosion formerLLArg = getLoweredExplosion(methodArg);
//   for (llvm::Value *val : formerLLArg.claimAll()) {
//      auto phi = cast<llvm::PHINode>(val);
//      assert(phi->getNumIncomingValues() == 0 && "phi already used");
//      phi->removeFromParent();
//      delete phi;
//   }
//   LoweredValues.erase(methodArg);
//
//   // Replace the lowered value with an ObjCMethod lowering.
////   setLoweredObjCMethod(methodArg, i->getMember());
//
//   // Create the branch.
//   Builder.CreateCondBr(call, hasMethodBB.bb, noMethodBB.bb);
}

void IRGenPILFunction::visitBranchInst(polar::BranchInst *i) {
   LoweredBB &lbb = getLoweredBB(i->getDestBB());
   addIncomingPILArgumentsToPHINodes(*this, lbb, i->getArgs());
   Builder.CreateBr(lbb.bb);
}

void IRGenPILFunction::visitCondBranchInst(polar::CondBranchInst *i) {
   LoweredBB &trueBB = getLoweredBB(i->getTrueBB());
   LoweredBB &falseBB = getLoweredBB(i->getFalseBB());
   llvm::Value *condValue =
      getLoweredExplosion(i->getCondition()).claimNext();

   addIncomingPILArgumentsToPHINodes(*this, trueBB, i->getTrueArgs());
   addIncomingPILArgumentsToPHINodes(*this, falseBB, i->getFalseArgs());

   llvm::MDNode *Weights = nullptr;
   auto TrueBBCount = i->getTrueBBCount();
   auto FalseBBCount = i->getFalseBBCount();
   if (TrueBBCount || FalseBBCount)
      Weights = IGM.createProfileWeights(TrueBBCount ? TrueBBCount.getValue() : 0,
                                         FalseBBCount ? FalseBBCount.getValue() : 0);

   Builder.CreateCondBr(condValue, trueBB.bb, falseBB.bb, Weights);
}

void IRGenPILFunction::visitRetainValueInst(polar::RetainValueInst *i) {
   Explosion in = getLoweredExplosion(i->getOperand());
   Explosion out;
   cast<LoadableTypeInfo>(getTypeInfo(i->getOperand()->getType()))
      .copy(*this, in, out, i->isAtomic() ? irgen::Atomicity::Atomic
                                          : irgen::Atomicity::NonAtomic);
   (void)out.claimAll();
}

void IRGenPILFunction::visitRetainValueAddrInst(polar::RetainValueAddrInst *i) {
   assert(i->getAtomicity() == RefCountingInst::Atomicity::Atomic &&
          "Non atomic retains are not supported");
   PILValue operandValue = i->getOperand();
   Address addr = getLoweredAddress(operandValue);
   PILType addrTy = operandValue->getType();
   PILType objectT = addrTy.getObjectType();
   llvm::Type *llvmType = addr.getAddress()->getType();
   const TypeInfo &addrTI = getTypeInfo(addrTy);
   auto *outlinedF = IGM.getOrCreateRetainFunction(addrTI, objectT, llvmType);
   llvm::Value *args[] = {addr.getAddress()};
   llvm::CallInst *call = Builder.CreateCall(outlinedF, args);
   call->setCallingConv(IGM.DefaultCC);
}

void IRGenPILFunction::visitCopyValueInst(polar::CopyValueInst *i) {
   Explosion in = getLoweredExplosion(i->getOperand());
   Explosion out;
   cast<LoadableTypeInfo>(getTypeInfo(i->getOperand()->getType()))
      .copy(*this, in, out, getDefaultAtomicity());
   setLoweredExplosion(i, out);
}

// TODO: Implement this more generally for arbitrary values. Currently the
// PIL verifier restricts it to single-refcounted-pointer types.
void IRGenPILFunction::visitAutoreleaseValueInst(polar::AutoreleaseValueInst *i)
{
   /// TODO
//   Explosion in = getLoweredExplosion(i->getOperand());
//   auto val = in.claimNext();
//
//   emitObjCAutoreleaseCall(val);
}

void IRGenPILFunction::visitSetDeallocatingInst(SetDeallocatingInst *i) {
   auto *ARI = dyn_cast<AllocRefInst>(i->getOperand());
   if (ARI && StackAllocs.count(ARI)) {
      // A small peep-hole optimization: If the operand is allocated on stack and
      // there is no "significant" code between the set_deallocating and the final
      // dealloc_ref, the set_deallocating is not required.
      //   %0 = alloc_ref [stack]
      //     ...
      //   set_deallocating %0 // not needed
      //     // code which does not depend on the RC_DEALLOCATING_FLAG flag.
      //   dealloc_ref %0      // not needed (stems from the inlined deallocator)
      //     ...
      //   dealloc_ref [stack] %0
      PILBasicBlock::iterator Iter(i);
      PILBasicBlock::iterator End = i->getParent()->end();
      for (++Iter; Iter != End; ++Iter) {
         PILInstruction *I = &*Iter;
         if (auto *DRI = dyn_cast<DeallocRefInst>(I)) {
            if (DRI->getOperand() == ARI) {
               // The set_deallocating is followed by a dealloc_ref -> we can ignore
               // it.
               return;
            }
         }
         // Assume that any instruction with side-effects may depend on the
         // RC_DEALLOCATING_FLAG flag.
         if (I->mayHaveSideEffects())
            break;
      }
   }
   Explosion lowered = getLoweredExplosion(i->getOperand());
   emitNativeSetDeallocating(lowered.claimNext());
}

void IRGenPILFunction::visitReleaseValueInst(polar::ReleaseValueInst *i) {
   Explosion in = getLoweredExplosion(i->getOperand());
   cast<LoadableTypeInfo>(getTypeInfo(i->getOperand()->getType()))
      .consume(*this, in, i->isAtomic() ? irgen::Atomicity::Atomic
                                        : irgen::Atomicity::NonAtomic);
}

void IRGenPILFunction::visitReleaseValueAddrInst(
   polar::ReleaseValueAddrInst *i) {
   assert(i->getAtomicity() == RefCountingInst::Atomicity::Atomic &&
          "Non atomic retains are not supported");
   PILValue operandValue = i->getOperand();
   Address addr = getLoweredAddress(operandValue);
   PILType addrTy = operandValue->getType();
   PILType objectT = addrTy.getObjectType();
   llvm::Type *llvmType = addr.getAddress()->getType();
   const TypeInfo &addrTI = getTypeInfo(addrTy);
   auto *outlinedF = IGM.getOrCreateReleaseFunction(
      addrTI, objectT, llvmType);
   llvm::Value *args[] = {addr.getAddress()};
   llvm::CallInst *call = Builder.CreateCall(outlinedF, args);
   call->setCallingConv(IGM.DefaultCC);
}

void IRGenPILFunction::visitDestroyValueInst(polar::DestroyValueInst *i) {
   Explosion in = getLoweredExplosion(i->getOperand());
   cast<LoadableTypeInfo>(getTypeInfo(i->getOperand()->getType()))
      .consume(*this, in, getDefaultAtomicity());
}

void IRGenPILFunction::visitStructInst(polar::StructInst *i) {
   Explosion out;
   for (PILValue elt : i->getElements())
      out.add(getLoweredExplosion(elt).claimAll());
   setLoweredExplosion(i, out);
}

void IRGenPILFunction::visitTupleInst(polar::TupleInst *i) {
   Explosion out;
   for (PILValue elt : i->getElements())
      out.add(getLoweredExplosion(elt).claimAll());
   setLoweredExplosion(i, out);
}

void IRGenPILFunction::visitEnumInst(polar::EnumInst *i) {
   Explosion data = (i->hasOperand())
                    ? getLoweredExplosion(i->getOperand())
                    : Explosion();
   Explosion out;
   emitInjectLoadableEnum(*this, i->getType(), i->getElement(), data, out);
   setLoweredExplosion(i, out);
}

void IRGenPILFunction::visitInitEnumDataAddrInst(polar::InitEnumDataAddrInst *i) {
   Address enumAddr = getLoweredAddress(i->getOperand());
   Address dataAddr = emitProjectEnumAddressForStore(*this,
                                                     i->getOperand()->getType(),
                                                     enumAddr,
                                                     i->getElement());
   setLoweredAddress(i, dataAddr);
}

void IRGenPILFunction::visitUncheckedEnumDataInst(polar::UncheckedEnumDataInst *i) {
   Explosion enumVal = getLoweredExplosion(i->getOperand());
   Explosion data;
   emitProjectLoadableEnum(*this, i->getOperand()->getType(),
                           enumVal, i->getElement(), data);
   setLoweredExplosion(i, data);
}

void IRGenPILFunction::visitUncheckedTakeEnumDataAddrInst(polar::UncheckedTakeEnumDataAddrInst *i) {
   Address enumAddr = getLoweredAddress(i->getOperand());
   Address dataAddr = emitDestructiveProjectEnumAddressForLoad(*this,
                                                               i->getOperand()->getType(),
                                                               enumAddr,
                                                               i->getElement());
   setLoweredAddress(i, dataAddr);
}

void IRGenPILFunction::visitInjectEnumAddrInst(polar::InjectEnumAddrInst *i) {
   Address enumAddr = getLoweredAddress(i->getOperand());
   emitStoreEnumTagToAddress(*this, i->getOperand()->getType(),
                             enumAddr, i->getElement());
}

void IRGenPILFunction::visitTupleExtractInst(polar::TupleExtractInst *i) {
   Explosion fullTuple = getLoweredExplosion(i->getOperand());
   Explosion output;
   PILType baseType = i->getOperand()->getType();

   projectTupleElementFromExplosion(*this,
                                    baseType,
                                    fullTuple,
                                    i->getFieldNo(),
                                    output);
   (void)fullTuple.claimAll();
   setLoweredExplosion(i, output);
}

void IRGenPILFunction::visitTupleElementAddrInst(polar::TupleElementAddrInst *i)
{
   Address base = getLoweredAddress(i->getOperand());
   PILType baseType = i->getOperand()->getType();

   Address field = projectTupleElementAddress(*this, base, baseType,
                                              i->getFieldNo());
   setLoweredAddress(i, field);
}

void IRGenPILFunction::visitStructExtractInst(polar::StructExtractInst *i) {
   Explosion operand = getLoweredExplosion(i->getOperand());
   Explosion lowered;
   PILType baseType = i->getOperand()->getType();

   projectPhysicalStructMemberFromExplosion(*this,
                                            baseType,
                                            operand,
                                            i->getField(),
                                            lowered);

   (void)operand.claimAll();
   setLoweredExplosion(i, lowered);
}

void IRGenPILFunction::visitStructElementAddrInst(
   polar::StructElementAddrInst *i) {
   Address base = getLoweredAddress(i->getOperand());
   PILType baseType = i->getOperand()->getType();

   Address field = projectPhysicalStructMemberAddress(*this, base, baseType,
                                                      i->getField());
   setLoweredAddress(i, field);
}

void IRGenPILFunction::visitRefElementAddrInst(polar::RefElementAddrInst *i) {
   Explosion base = getLoweredExplosion(i->getOperand());
   llvm::Value *value = base.claimNext();

   PILType baseTy = i->getOperand()->getType();
   Address field = projectPhysicalClassMemberAddress(*this, value, baseTy,
                                                     i->getType(), i->getField())
      .getAddress();
   setLoweredAddress(i, field);
}

void IRGenPILFunction::visitRefTailAddrInst(RefTailAddrInst *i) {
   PILValue Ref = i->getOperand();
   llvm::Value *RefValue = getLoweredExplosion(Ref).claimNext();

   Address TailAddr = emitTailProjection(*this, RefValue, Ref->getType(),
                                         i->getTailType());
   setLoweredAddress(i, TailAddr);
}

static bool isInvariantAddress(PILValue v) {
   auto root = getUnderlyingAddressRoot(v);
   if (auto ptrRoot = dyn_cast<PointerToAddressInst>(root)) {
      return ptrRoot->isInvariant();
   }
   // TODO: We could be more aggressive about considering addresses based on
   // `let` variables as invariant when the type of the address is known not to
   // have any sharably-mutable interior storage (in other words, no weak refs,
   // atomics, etc.)
   return false;
}

void IRGenPILFunction::visitLoadInst(polar::LoadInst *i) {
   Explosion lowered;
   Address source = getLoweredAddress(i->getOperand());
   PILType objType = i->getType().getObjectType();
   const auto &typeInfo = cast<LoadableTypeInfo>(getTypeInfo(objType));

   switch (i->getOwnershipQualifier()) {
      case LoadOwnershipQualifier::Unqualified:
      case LoadOwnershipQualifier::Trivial:
      case LoadOwnershipQualifier::Take:
         typeInfo.loadAsTake(*this, source, lowered);
         break;
      case LoadOwnershipQualifier::Copy:
         typeInfo.loadAsCopy(*this, source, lowered);
         break;
   }

   if (isInvariantAddress(i->getOperand())) {
      // It'd be better to push this down into `loadAs` methods, perhaps...
      for (auto value : lowered.getAll())
         if (auto load = dyn_cast<llvm::LoadInst>(value))
            setInvariantLoad(load);
   }
   setLoweredExplosion(i, lowered);
}

void IRGenPILFunction::visitStoreInst(polar::StoreInst *i) {
   Explosion source = getLoweredExplosion(i->getSrc());
   Address dest = getLoweredAddress(i->getDest());
   PILType objType = i->getSrc()->getType().getObjectType();

   const auto &typeInfo = cast<LoadableTypeInfo>(getTypeInfo(objType));
   switch (i->getOwnershipQualifier()) {
      case StoreOwnershipQualifier::Unqualified:
      case StoreOwnershipQualifier::Init:
      case StoreOwnershipQualifier::Trivial:
         typeInfo.initialize(*this, source, dest, false);
         break;
      case StoreOwnershipQualifier::Assign:
         typeInfo.assign(*this, source, dest, false);
         break;
   }
}

/// Emit the artificial error result argument.
void IRGenPILFunction::emitErrorResultVar(CanPILFunctionType FnTy,
                                          PILResultInfo ErrorInfo,
                                          DebugValueInst *DbgValue) {
   // We don't need a shadow error variable for debugging on ABI's that return
   // swifterror in a register.
   if (IGM.IsSwiftErrorInRegister)
      return;
   auto ErrorResultSlot = getErrorResultSlot(
      IGM.silConv.getPILType(ErrorInfo, FnTy));
   auto Var = DbgValue->getVarInfo();
   assert(Var && "error result without debug info");
   auto Storage = emitShadowCopyIfNeeded(ErrorResultSlot.getAddress(),
                                         getDebugScope(), *Var, false);
   if (!IGM.DebugInfo)
      return;
   auto DbgTy = DebugTypeInfo::getErrorResult(
      ErrorInfo.getReturnValueType(IGM.getPILModule(), FnTy),
      ErrorResultSlot->getType(),
      IGM.getPointerSize(), IGM.getPointerAlignment());
   IGM.DebugInfo->emitVariableDeclaration(Builder, Storage, DbgTy,
                                          getDebugScope(), nullptr, *Var,
                                          IndirectValue, ArtificialValue);
}

void IRGenPILFunction::visitDebugValueInst(DebugValueInst *i) {
   if (i->getDebugScope()->getInlinedFunction()->isTransparent())
      return;

   auto VarInfo = i->getVarInfo();
   assert(VarInfo && "debug_value without debug info");
   auto PILVal = i->getOperand();
   if (isa<PILUndef>(PILVal)) {
      // We cannot track the location of inlined error arguments because it has no
      // representation in PIL.
      if (!i->getDebugScope()->InlinedCallSite && VarInfo->Name == "$error") {
         auto funcTy = CurPILFn->getLoweredFunctionType();
         emitErrorResultVar(funcTy, funcTy->getErrorResult(), i);
      }
      return;
   }

   bool IsAnonymous = false;
   VarInfo->Name = getVarName(i, IsAnonymous);
   DebugTypeInfo DbgTy;
   PILType PILTy = PILVal->getType();
   auto RealTy = PILVal->getType().getAstType();
   if (VarDecl *Decl = i->getDecl()) {
      DbgTy = DebugTypeInfo::getLocalVariable(
         Decl, RealTy, getTypeInfo(PILVal->getType()));
   } else if (i->getFunction()->isBare() &&
              !PILTy.hasArchetype() && !VarInfo->Name.empty()) {
      // Preliminary support for .sil debug information.
      DbgTy = DebugTypeInfo::getFromTypeInfo(RealTy, getTypeInfo(PILTy));
   } else
      return;

   // Put the value into a stack slot at -Onone.
   llvm::SmallVector<llvm::Value *, 8> Copy;
   emitShadowCopyIfNeeded(PILVal, i->getDebugScope(), *VarInfo, IsAnonymous,
                          Copy);
   bindArchetypes(DbgTy.getType());
   if (!IGM.DebugInfo)
      return;

   emitDebugVariableDeclaration(Copy, DbgTy, PILTy, i->getDebugScope(),
                                i->getDecl(), *VarInfo);
}

void IRGenPILFunction::visitDebugValueAddrInst(DebugValueAddrInst *i) {
   if (i->getDebugScope()->getInlinedFunction()->isTransparent())
      return;

   VarDecl *Decl = i->getDecl();
   if (!Decl)
      return;

   auto PILVal = i->getOperand();
   if (isa<PILUndef>(PILVal))
      return;

   auto VarInfo = i->getVarInfo();
   assert(VarInfo && "debug_value_addr without debug info");
   bool IsAnonymous = false;
   bool IsLoadablyByAddress = isa<AllocStackInst>(PILVal);
   VarInfo->Name = getVarName(i, IsAnonymous);
   auto Addr = getLoweredAddress(PILVal).getAddress();
   PILType PILTy = PILVal->getType();
   auto RealType = PILTy.getAstType();

   auto DbgTy = DebugTypeInfo::getLocalVariable(
      Decl, RealType, getTypeInfo(PILVal->getType()));
   bindArchetypes(DbgTy.getType());
   if (!IGM.DebugInfo)
      return;

   // Put the value's address into a stack slot at -Onone and emit a debug
   // intrinsic.
   emitDebugVariableDeclaration(
      emitShadowCopyIfNeeded(Addr, i->getDebugScope(), *VarInfo, IsAnonymous),
      DbgTy, PILType(), i->getDebugScope(), Decl, *VarInfo,
      (IsLoadablyByAddress) ? DirectValue : IndirectValue);
}

void IRGenPILFunction::visitFixLifetimeInst(polar::FixLifetimeInst *i) {
   if (i->getOperand()->getType().isAddress()) {
      // Just pass in the address to fix lifetime if we have one. We will not do
      // anything to it so nothing bad should happen.
      emitFixLifetime(getLoweredAddress(i->getOperand()).getAddress());
      return;
   }

   // Handle objects.
   Explosion in = getLoweredExplosion(i->getOperand());
   cast<LoadableTypeInfo>(getTypeInfo(i->getOperand()->getType()))
      .fixLifetime(*this, in);
}

void IRGenPILFunction::visitMarkDependenceInst(polar::MarkDependenceInst *i) {
   // Dependency-marking is purely for PIL.  Just forward the input as
   // the result.

   PILValue value = i->getValue();
   if (value->getType().isAddress()) {
      setLoweredAddress(i, getLoweredAddress(value));
   } else {
      Explosion temp = getLoweredExplosion(value);
      setLoweredExplosion(i, temp);
   }
}

void IRGenPILFunction::visitCopyBlockInst(CopyBlockInst *i) {
   /// TODO
   Explosion lowered = getLoweredExplosion(i->getOperand());
//   llvm::Value *copied = emitBlockCopyCall(lowered.claimNext());
   Explosion result;
//   result.add(copied);
   setLoweredExplosion(i, result);
}

void IRGenPILFunction::visitStrongRetainInst(polar::StrongRetainInst *i) {
   Explosion lowered = getLoweredExplosion(i->getOperand());
   auto &ti = cast<ReferenceTypeInfo>(getTypeInfo(i->getOperand()->getType()));
   ti.strongRetain(*this, lowered, i->isAtomic() ? irgen::Atomicity::Atomic
                                                 : irgen::Atomicity::NonAtomic);
}

void IRGenPILFunction::visitStrongReleaseInst(polar::StrongReleaseInst *i) {
   Explosion lowered = getLoweredExplosion(i->getOperand());
   auto &ti = cast<ReferenceTypeInfo>(getTypeInfo(i->getOperand()->getType()));
   ti.strongRelease(*this, lowered, i->isAtomic() ? irgen::Atomicity::Atomic
                                                  : irgen::Atomicity::NonAtomic);
}

/// Given a PILType which is a ReferenceStorageType, return the type
/// info for the underlying reference type.
static const ReferenceTypeInfo &getReferentTypeInfo(IRGenFunction &IGF,
                                                    PILType silType) {
   auto type = silType.castTo<ReferenceStorageType>().getReferentType();
   if (auto ty = type->getOptionalObjectType())
      type = ty->getCanonicalType();
   return cast<ReferenceTypeInfo>(IGF.getTypeInfoForLowered(type));
}

#define NEVER_LOADABLE_CHECKED_REF_STORAGE(Name, name, ...) \
  void IRGenPILFunction::visitLoad##Name##Inst(polar::Load##Name##Inst *i) { \
    Address source = getLoweredAddress(i->getOperand()); \
    auto silTy = i->getOperand()->getType(); \
    auto ty = cast<Name##StorageType>(silTy.getAstType()); \
    auto isOptional = bool(ty.getReferentType()->getOptionalObjectType()); \
    auto &ti = getReferentTypeInfo(*this, silTy); \
    Explosion result; \
    if (i->isTake()) { \
      ti.name##TakeStrong(*this, source, result, isOptional); \
    } else { \
      ti.name##LoadStrong(*this, source, result, isOptional); \
    } \
    setLoweredExplosion(i, result); \
  } \
  void IRGenPILFunction::visitStore##Name##Inst(polar::Store##Name##Inst *i) { \
    Explosion source = getLoweredExplosion(i->getSrc()); \
    Address dest = getLoweredAddress(i->getDest()); \
    auto silTy = i->getDest()->getType(); \
    auto ty = cast<Name##StorageType>(silTy.getAstType()); \
    auto isOptional = bool(ty.getReferentType()->getOptionalObjectType()); \
    auto &ti = getReferentTypeInfo(*this, silTy); \
    if (i->isInitializationOfDest()) { \
      ti.name##Init(*this, source, dest, isOptional); \
    } else { \
      ti.name##Assign(*this, source, dest, isOptional); \
    } \
  }
#define ALWAYS_LOADABLE_CHECKED_REF_STORAGE(Name, name, ...)                   \
  void IRGenPILFunction::visitStrongRetain##Name##Inst(                        \
      polar::StrongRetain##Name##Inst *i) {                                    \
    Explosion lowered = getLoweredExplosion(i->getOperand());                  \
    auto &ti = getReferentTypeInfo(*this, i->getOperand()->getType());         \
    ti.strongRetain##Name(*this, lowered,                                      \
                          i->isAtomic() ? irgen::Atomicity::Atomic             \
                                        : irgen::Atomicity::NonAtomic);        \
  }                                                                            \
  void IRGenPILFunction::visit##Name##RetainInst(polar::Name##RetainInst *i) { \
    Explosion lowered = getLoweredExplosion(i->getOperand());                  \
    auto &ti = getReferentTypeInfo(*this, i->getOperand()->getType());         \
    ti.name##Retain(*this, lowered,                                            \
                    i->isAtomic() ? irgen::Atomicity::Atomic                   \
                                  : irgen::Atomicity::NonAtomic);              \
  }                                                                            \
  void IRGenPILFunction::visit##Name##ReleaseInst(                             \
      polar::Name##ReleaseInst *i) {                                           \
    Explosion lowered = getLoweredExplosion(i->getOperand());                  \
    auto &ti = getReferentTypeInfo(*this, i->getOperand()->getType());         \
    ti.name##Release(*this, lowered,                                           \
                     i->isAtomic() ? irgen::Atomicity::Atomic                  \
                                   : irgen::Atomicity::NonAtomic);             \
  }                                                                            \
  void IRGenPILFunction::visitStrongCopy##Name##ValueInst(                     \
      polar::StrongCopy##Name##ValueInst *i) {                                 \
    Explosion in = getLoweredExplosion(i->getOperand());                       \
    auto silTy = i->getOperand()->getType();                                   \
    auto ty = cast<Name##StorageType>(silTy.getAstType());                     \
    auto isOptional = bool(ty.getReferentType()->getOptionalObjectType());     \
    auto &ti = getReferentTypeInfo(*this, silTy);                              \
    ti.strongRetain##Name(*this, in, irgen::Atomicity::Atomic);                \
    /* Semantically we are just passing through the input parameter but as a   \
     */                                                                        \
    /* strong reference... at LLVM IR level these type differences don't */    \
    /* matter. So just set the lowered explosion appropriately. */             \
    Explosion output = getLoweredExplosion(i->getOperand());                   \
    if (isOptional) {                                                          \
      auto values = output.claimAll();                                         \
      output.reset();                                                          \
      for (auto value : values) {                                              \
        output.add(Builder.CreatePtrToInt(value, IGM.IntPtrTy));               \
      }                                                                        \
    }                                                                          \
    setLoweredExplosion(i, output);                                            \
  }
#define SOMETIMES_LOADABLE_CHECKED_REF_STORAGE(Name, name, ...) \
  NEVER_LOADABLE_CHECKED_REF_STORAGE(Name, name, "...") \
  ALWAYS_LOADABLE_CHECKED_REF_STORAGE(Name, name, "...")
#define UNCHECKED_REF_STORAGE(Name, name, ...)                                 \
  void IRGenPILFunction::visitStrongCopy##Name##ValueInst(                     \
      polar::StrongCopy##Name##ValueInst *i) {                                 \
    Explosion in = getLoweredExplosion(i->getOperand());                       \
    auto silTy = i->getOperand()->getType();                                   \
    auto ty = cast<Name##StorageType>(silTy.getAstType());                     \
    auto isOptional = bool(ty.getReferentType()->getOptionalObjectType());     \
    auto &ti = getReferentTypeInfo(*this, silTy);                              \
    /* Since we are unchecked, we just use strong retain here. We do not       \
     * perform any checks */                                                   \
    ti.strongRetain(*this, in, irgen::Atomicity::Atomic);                      \
    /* Semantically we are just passing through the input parameter but as a   \
     */                                                                        \
    /* strong reference... at LLVM IR level these type differences don't */    \
    /* matter. So just set the lowered explosion appropriately. */             \
    Explosion output = getLoweredExplosion(i->getOperand());                   \
    if (isOptional) {                                                          \
      auto values = output.claimAll();                                         \
      output.reset();                                                          \
      for (auto value : values) {                                              \
        output.add(Builder.CreatePtrToInt(value, IGM.IntPtrTy));               \
      }                                                                        \
    }                                                                          \
    setLoweredExplosion(i, output);                                            \
  }
#include "polarphp/ast/ReferenceStorageDef.h"
#undef COMMON_CHECKED_REF_STORAGE

static bool hasReferenceSemantics(IRGenPILFunction &IGF,
                                  PILType silType) {
   auto operType = silType.getAstType();
   auto valueType = operType->getOptionalObjectType();
   auto objType = valueType ? valueType : operType;
   return (objType->mayHaveSuperclass()
           || objType->isClassExistentialType()
           || objType->is<BuiltinNativeObjectType>()
           || objType->is<BuiltinBridgeObjectType>());
}

static llvm::Value *emitIsUnique(IRGenPILFunction &IGF, PILValue operand,
                                 SourceLoc loc) {
   if (!hasReferenceSemantics(IGF, operand->getType())) {
      IGF.emitTrap("isUnique called for a non-reference", /*EmitUnreachable=*/false);
      return llvm::UndefValue::get(IGF.IGM.Int1Ty);
   }

   auto &operTI = cast<LoadableTypeInfo>(IGF.getTypeInfo(operand->getType()));
   LoadedRef ref =
      operTI.loadRefcountedPtr(IGF, loc, IGF.getLoweredAddress(operand));

   return
      IGF.emitIsUniqueCall(ref.getValue(), loc, ref.isNonNull());
}

void IRGenPILFunction::visitIsUniqueInst(polar::IsUniqueInst *i) {
   llvm::Value *result = emitIsUnique(*this, i->getOperand(),
                                      i->getLoc().getSourceLoc());
   Explosion out;
   out.add(result);
   setLoweredExplosion(i, out);
}

void IRGenPILFunction::visitIsEscapingClosureInst(
   polar::IsEscapingClosureInst *i) {
   // The closure operand is allowed to be an optional closure.
   auto operandType = i->getOperand()->getType();
   if (operandType.getOptionalObjectType())
      operandType = operandType.getOptionalObjectType();

   auto fnType = operandType.getAs<PILFunctionType>();
   assert(fnType->getExtInfo().hasContext() && "Must have a closure operand");
   (void)fnType;

   // This code relies on that an optional<()->()>'s tag fits in the function
   // pointer.
   auto &TI = cast<LoadableTypeInfo>(getTypeInfo(operandType));
   assert(TI.mayHaveExtraInhabitants(IGM) &&
          "Must have extra inhabitants to be able to handle the optional "
          "closure case");
   (void)TI;

   Explosion closure = getLoweredExplosion(i->getOperand());
   auto func = closure.claimNext();
   (void)func;
   auto context = closure.claimNext();
   assert(closure.empty());
   if (context->getType()->isIntegerTy())
      context = Builder.CreateIntToPtr(context, IGM.RefCountedPtrTy);
   auto result = emitIsEscapingClosureCall(context, i->getLoc().getSourceLoc(),
                                           i->getVerificationType());
   Explosion out;
   out.add(result);
   setLoweredExplosion(i, out);
}

void IRGenPILFunction::emitDebugInfoForAllocStack(AllocStackInst *i,
                                                  const TypeInfo &type,
                                                  llvm::Value *addr) {
   auto VarInfo = i->getVarInfo();
   if (!VarInfo)
      return;

   VarDecl *Decl = i->getDecl();
   // Describe the underlying alloca. This way an llvm.dbg.declare instrinsic
   // is used, which is valid for the entire lifetime of the alloca.
   if (auto *BitCast = dyn_cast<llvm::BitCastInst>(addr)) {
      auto *Op0 = BitCast->getOperand(0);
      if (auto *Alloca = dyn_cast<llvm::AllocaInst>(Op0))
         addr = Alloca;
      else if (auto *CoroAllocaGet = dyn_cast<llvm::IntrinsicInst>(Op0)) {
         if (CoroAllocaGet->getIntrinsicID() == llvm::Intrinsic::coro_alloca_get)
            addr = CoroAllocaGet;
      }
   }

   auto DS = i->getDebugScope();
   if (!DS)
      return;

   if (i->getDebugScope()->getInlinedFunction()->isTransparent())
      return;

   bool IsAnonymous = false;
   VarInfo->Name = getVarName(i, IsAnonymous);

   // At this point addr must be an alloca or an undef.
   assert(isa<llvm::AllocaInst>(addr) || isa<llvm::UndefValue>(addr) ||
          isa<llvm::IntrinsicInst>(addr));

   auto Indirection = DirectValue;
   if (!IGM.IRGen.Opts.DisableDebuggerShadowCopies &&
       !IGM.IRGen.Opts.shouldOptimize())
      if (auto *Alloca = dyn_cast<llvm::AllocaInst>(addr))
         if (!Alloca->isStaticAlloca()) {
            // Store the address of the dynamic alloca on the stack.
            addr = emitShadowCopy(addr, DS, *VarInfo, IGM.getPointerAlignment());
            Indirection = IndirectValue;
         }

   if (!Decl)
      return;

   // Ignore compiler-generated patterns but not optional bindings.
   if (auto *Pattern = Decl->getParentPattern())
      if (Pattern->isImplicit() &&
          Pattern->getKind() != PatternKind::OptionalSome)
         return;

   PILType PILTy = i->getType();
   auto RealType = PILTy.getAstType();
   auto DbgTy = DebugTypeInfo::getLocalVariable(Decl, RealType, type);

   bindArchetypes(DbgTy.getType());
   if (IGM.DebugInfo)
      emitDebugVariableDeclaration(addr, DbgTy, PILTy, DS, Decl, *VarInfo,
                                   Indirection);
}

void IRGenPILFunction::visitAllocStackInst(polar::AllocStackInst *i) {
   const TypeInfo &type = getTypeInfo(i->getElementType());

   // Derive name from PIL location.
   StringRef dbgname;
   VarDecl *Decl = i->getDecl();
# ifndef NDEBUG
   // If this is a DEBUG build, use pretty names for the LLVM IR.
   bool IsAnonymous = false;
   dbgname = getVarName(i, IsAnonymous);
# endif

   auto addr = type.allocateStack(*this, i->getElementType(), dbgname);
   setLoweredStackAddress(i, addr);

   // Generate Debug Info.
   if (!Decl)
      return;

   assert(i->getVarInfo() && "alloc_stack without debug info");

   Type Desugared = Decl->getType()->getDesugaredType();
   if (Desugared->getClassOrBoundGenericClass() ||
       Desugared->getStructOrBoundGenericStruct())
      zeroInit(dyn_cast<llvm::AllocaInst>(addr.getAddress().getAddress()));
   emitDebugInfoForAllocStack(i, type, addr.getAddress().getAddress());
}

static void
buildTailArrays(IRGenPILFunction &IGF,
                SmallVectorImpl<std::pair<PILType, llvm::Value *>> &TailArrays,
                AllocRefInstBase *ARI) {
   auto Types = ARI->getTailAllocatedTypes();
   auto Counts = ARI->getTailAllocatedCounts();
   for (unsigned Idx = 0, NumTypes = Types.size(); Idx < NumTypes; ++Idx) {
      Explosion ElemCount = IGF.getLoweredExplosion(Counts[Idx].get());
      TailArrays.push_back({Types[Idx], ElemCount.claimNext()});
   }
}

void IRGenPILFunction::visitAllocRefInst(polar::AllocRefInst *i) {
   int StackAllocSize = -1;
   if (i->canAllocOnStack()) {
      estimateStackSize();
      // Is there enough space for stack allocation?
      StackAllocSize = IGM.IRGen.Opts.StackPromotionSizeLimit - EstimatedStackSize;
   }
   SmallVector<std::pair<PILType, llvm::Value *>, 4> TailArrays;
   buildTailArrays(*this, TailArrays, i);

   llvm::Value *alloced = emitClassAllocation(*this, i->getType(), i->isObjC(),
                                              StackAllocSize, TailArrays);
   if (StackAllocSize >= 0) {
      // Remember that this alloc_ref allocates the object on the stack.

      StackAllocs.insert(i);
      EstimatedStackSize += StackAllocSize;
   }
   Explosion e;
   e.add(alloced);
   setLoweredExplosion(i, e);
}

void IRGenPILFunction::visitAllocRefDynamicInst(polar::AllocRefDynamicInst *i) {
   SmallVector<std::pair<PILType, llvm::Value *>, 4> TailArrays;
   buildTailArrays(*this, TailArrays, i);

   Explosion metadata = getLoweredExplosion(i->getMetatypeOperand());
   auto metadataValue = metadata.claimNext();
   llvm::Value *alloced = emitClassAllocationDynamic(*this, metadataValue,
                                                     i->getType(), i->isObjC(),
                                                     TailArrays);
   Explosion e;
   e.add(alloced);
   setLoweredExplosion(i, e);
}

void IRGenPILFunction::visitDeallocStackInst(polar::DeallocStackInst *i) {
   if (auto *closure = dyn_cast<PartialApplyInst>(i->getOperand())) {
      assert(closure->isOnStack());
      auto stackAddr = LoweredPartialApplyAllocations[i->getOperand()];
      emitDeallocateDynamicAlloca(stackAddr);
      return;
   }

   auto allocatedType = i->getOperand()->getType();
   const TypeInfo &allocatedTI = getTypeInfo(allocatedType);
   StackAddress stackAddr = getLoweredStackAddress(i->getOperand());

   allocatedTI.deallocateStack(*this, stackAddr, allocatedType);
}

void IRGenPILFunction::visitDeallocRefInst(polar::DeallocRefInst *i) {
   // Lower the operand.
   Explosion self = getLoweredExplosion(i->getOperand());
   auto selfValue = self.claimNext();
   auto *ARI = dyn_cast<AllocRefInst>(i->getOperand());
   if (!i->canAllocOnStack()) {
      if (ARI && StackAllocs.count(ARI)) {
         // We can ignore dealloc_refs (without [stack]) for stack allocated
         // objects.
         //
         //   %0 = alloc_ref [stack]
         //     ...
         //   dealloc_ref %0     // not needed (stems from the inlined deallocator)
         //     ...
         //   dealloc_ref [stack] %0
         return;
      }

      auto classType = i->getOperand()->getType();
      emitClassDeallocation(*this, classType, selfValue);
      return;
   }
   // It's a dealloc_ref [stack]. Even if the alloc_ref did not allocate the
   // object on the stack, we don't have to deallocate it, because it is
   // deallocated in the final release.
   assert(ARI->canAllocOnStack());
   if (StackAllocs.count(ARI)) {
      if (IGM.IRGen.Opts.EmitStackPromotionChecks) {
         selfValue = Builder.CreateBitCast(selfValue, IGM.RefCountedPtrTy);
         emitVerifyEndOfLifetimeCall(selfValue);
      } else {
         // This has two purposes:
         // 1. Tell LLVM the lifetime of the allocated stack memory.
         // 2. Avoid tail-call optimization which may convert the call to the final
         //    release to a jump, which is done after the stack frame is
         //    destructed.
         Builder.CreateLifetimeEnd(selfValue);
      }
   }
}

void IRGenPILFunction::visitDeallocPartialRefInst(polar::DeallocPartialRefInst *i) {
   Explosion self = getLoweredExplosion(i->getInstance());
   auto selfValue = self.claimNext();
   Explosion metadata = getLoweredExplosion(i->getMetatype());
   auto metadataValue = metadata.claimNext();
   auto classType = i->getInstance()->getType();

   emitPartialClassDeallocation(*this, classType, selfValue, metadataValue);
}

void IRGenPILFunction::visitDeallocBoxInst(polar::DeallocBoxInst *i) {
   Explosion owner = getLoweredExplosion(i->getOperand());
   llvm::Value *ownerPtr = owner.claimNext();

   auto boxTy = i->getOperand()->getType().castTo<PILBoxType>();
   emitDeallocateBox(*this, ownerPtr, boxTy);
}

void IRGenPILFunction::visitAllocBoxInst(polar::AllocBoxInst *i) {
   assert(i->getBoxType()->getLayout()->getFields().size() == 1
          && "multi field boxes not implemented yet");
   const TypeInfo &type = getTypeInfo(
      getPILBoxFieldType(IGM.getMaximalTypeExpansionContext(), i->getBoxType(),
                         IGM.getPILModule().Types, 0));

   // Derive name from PIL location.
   bool IsAnonymous = false;
   VarDecl *Decl = i->getDecl();
   StringRef Name = getVarName(i, IsAnonymous);
   StringRef DbgName =
# ifndef NDEBUG
      // If this is a DEBUG build, use pretty names for the LLVM IR.
      Name;
# else
   "";
# endif

   auto boxTy = i->getType().castTo<PILBoxType>();
   OwnedAddress boxWithAddr = emitAllocateBox(*this, boxTy,
                                              CurPILFn->getGenericEnvironment(),
                                              DbgName);
   setLoweredBox(i, boxWithAddr);

   if (i->getDebugScope()->getInlinedFunction()->isTransparent())
      return;

   if (!Decl)
      return;

   assert(i->getVarInfo() && "alloc_box without debug info");

   // FIXME: This is a workaround to not produce local variables for
   // capture list arguments like "[weak self]". The better solution
   // would be to require all variables to be described with a
   // PILDebugValue(Addr) and then not describe capture list
   // arguments.
   if (Name == IGM.Context.Id_self.str())
      return;

   assert(i->getBoxType()->getLayout()->getFields().size() == 1 &&
          "box for a local variable should only have one field");
   auto PILTy = getPILBoxFieldType(
      IGM.getMaximalTypeExpansionContext(),
      i->getBoxType(), IGM.getPILModule().Types, 0);
   auto RealType = PILTy.getAstType();
   auto DbgTy = DebugTypeInfo::getLocalVariable(Decl, RealType, type);

   auto VarInfo = i->getVarInfo();
   assert(VarInfo && "debug_value without debug info");

   auto Storage = emitShadowCopyIfNeeded(
      boxWithAddr.getAddress(), i->getDebugScope(), *VarInfo, IsAnonymous);

   if (!IGM.DebugInfo)
      return;

   IGM.DebugInfo->emitVariableDeclaration(Builder, Storage, DbgTy,
                                          i->getDebugScope(), Decl, *VarInfo,
                                          IndirectValue);
}

void IRGenPILFunction::visitProjectBoxInst(polar::ProjectBoxInst *i) {
   auto boxTy = i->getOperand()->getType().castTo<PILBoxType>();

   const LoweredValue &val = getLoweredValue(i->getOperand());
   if (val.isBoxWithAddress()) {
      // The operand is an alloc_box. We can directly reuse the address.
      setLoweredAddress(i, val.getAddressOfBox());
   } else {
      // The slow-path: we have to emit code to get from the box to it's
      // value address.
      Explosion box = val.getExplosion(*this, i->getOperand()->getType());
      auto addr = emitProjectBox(*this, box.claimNext(), boxTy);
      setLoweredAddress(i, addr);
   }
}

static ExclusivityFlags getExclusivityAction(PILAccessKind kind) {
   switch (kind) {
      case PILAccessKind::Read:
         return ExclusivityFlags::Read;
      case PILAccessKind::Modify:
         return ExclusivityFlags::Modify;
      case PILAccessKind::Init:
      case PILAccessKind::Deinit:
         llvm_unreachable("init/deinit access should not use dynamic enforcement");
   }
   llvm_unreachable("bad access kind");
}

static ExclusivityFlags getExclusivityFlags(PILModule &M,
                                            PILAccessKind kind,
                                            bool noNestedConflict) {
   auto flags = getExclusivityAction(kind);

   if (!noNestedConflict)
      flags |= ExclusivityFlags::Tracking;

   return flags;
}

static PILAccessEnforcement getEffectiveEnforcement(IRGenFunction &IGF,
                                                    BeginAccessInst *access) {
   auto enforcement = access->getEnforcement();

   // Don't use dynamic enforcement for known-empty types; there's no
   // actual memory there, and the address may not be valid and unique.
   // This is really a hack; we don't necessarily know that all clients
   // will agree whether a type is empty.  On the other hand, the situations
   // where IRGen generates a meaningless address should always be a subset
   // of cases where this triggers, because of the restrictions on abstracting
   // over addresses and the fact that we use static enforcement on inouts.
   if (enforcement == PILAccessEnforcement::Dynamic &&
       IGF.IGM.getTypeInfo(access->getSource()->getType())
          .isKnownEmpty(ResilienceExpansion::Maximal)) {
      enforcement = PILAccessEnforcement::Unsafe;
   }

   return enforcement;
}

template <class BeginAccessInst>
static ExclusivityFlags getExclusivityFlags(BeginAccessInst *i) {
   return getExclusivityFlags(i->getModule(), i->getAccessKind(),
                              i->hasNoNestedConflict());
}

void IRGenPILFunction::visitBeginAccessInst(BeginAccessInst *access) {
   Address addr = getLoweredAddress(access->getOperand());
   switch (getEffectiveEnforcement(*this, access)) {
      case PILAccessEnforcement::Unknown:
         llvm_unreachable("unknown access enforcement in IRGen!");

      case PILAccessEnforcement::Static:
      case PILAccessEnforcement::Unsafe:
         // nothing to do
         setLoweredAddress(access, addr);
         return;

      case PILAccessEnforcement::Dynamic: {
         llvm::Value *scratch = createAlloca(IGM.getFixedBufferTy(),
                                             IGM.getPointerAlignment(),
                                             "access-scratch").getAddress();
         Builder.CreateLifetimeStart(scratch);

         llvm::Value *pointer =
            Builder.CreateBitCast(addr.getAddress(), IGM.Int8PtrTy);
         llvm::Value *flags =
            llvm::ConstantInt::get(IGM.SizeTy, uint64_t(getExclusivityFlags(access)));
         llvm::Value *pc = llvm::ConstantPointerNull::get(IGM.Int8PtrTy);
         auto call = Builder.CreateCall(IGM.getBeginAccessFn(),
                                        { pointer, scratch, flags, pc });
         call->setDoesNotThrow();

         setLoweredDynamicallyEnforcedAddress(access, addr, scratch);
         return;
      }
   }
   llvm_unreachable("bad access enforcement");
}

static bool hasBeenInlined(BeginUnpairedAccessInst *access) {
   // Check to see if the buffer is defined locally.
   return isa<AllocStackInst>(access->getBuffer());
}

void IRGenPILFunction::visitBeginUnpairedAccessInst(
   BeginUnpairedAccessInst *access) {
   Address addr = getLoweredAddress(access->getSource());
   switch (access->getEnforcement()) {
      case PILAccessEnforcement::Unknown:
         llvm_unreachable("unknown access enforcement in IRGen!");

      case PILAccessEnforcement::Static:
      case PILAccessEnforcement::Unsafe:
         // nothing to do
         return;

      case PILAccessEnforcement::Dynamic: {
         llvm::Value *scratch = getLoweredAddress(access->getBuffer()).getAddress();

         llvm::Value *pointer =
            Builder.CreateBitCast(addr.getAddress(), IGM.Int8PtrTy);
         llvm::Value *flags =
            llvm::ConstantInt::get(IGM.SizeTy, uint64_t(getExclusivityFlags(access)));

         // Compute the effective PC of the access.
         // Since begin_unpaired_access is designed for materializeForSet, our
         // heuristic here is as well: we've either been inlined, in which case
         // we should use the current PC (i.e. pass null), or we haven't,
         // in which case we should use the caller, which is generally ok because
         // materializeForSet can't usually be thunked.
         llvm::Value *pc;
         if (hasBeenInlined(access)) {
            pc = llvm::ConstantPointerNull::get(IGM.Int8PtrTy);
         } else {
            auto retAddrFn =
               llvm::Intrinsic::getDeclaration(IGM.getModule(),
                                               llvm::Intrinsic::returnaddress);
            pc = Builder.CreateCall(retAddrFn,
                                    { llvm::ConstantInt::get(IGM.Int32Ty, 0) });
         }

         auto call = Builder.CreateCall(IGM.getBeginAccessFn(),
                                        { pointer, scratch, flags, pc });
         call->setDoesNotThrow();
         return;
      }
   }
   llvm_unreachable("bad access enforcement");
}

void IRGenPILFunction::visitEndAccessInst(EndAccessInst *i) {
   auto access = i->getBeginAccess();
   switch (getEffectiveEnforcement(*this, access)) {
      case PILAccessEnforcement::Unknown:
         llvm_unreachable("unknown access enforcement in IRGen!");

      case PILAccessEnforcement::Static:
      case PILAccessEnforcement::Unsafe:
         // nothing to do
         return;

      case PILAccessEnforcement::Dynamic: {
         if (access->hasNoNestedConflict())
            return;

         auto scratch = getLoweredDynamicEnforcementScratchBuffer(access);

         auto call = Builder.CreateCall(IGM.getEndAccessFn(), { scratch });
         call->setDoesNotThrow();

         Builder.CreateLifetimeEnd(scratch);
         return;
      }
   }
   llvm_unreachable("bad access enforcement");
}

void IRGenPILFunction::visitEndUnpairedAccessInst(EndUnpairedAccessInst *i) {
   switch (i->getEnforcement()) {
      case PILAccessEnforcement::Unknown:
         llvm_unreachable("unknown access enforcement in IRGen!");

      case PILAccessEnforcement::Static:
      case PILAccessEnforcement::Unsafe:
         // nothing to do
         return;

      case PILAccessEnforcement::Dynamic: {
         auto scratch = getLoweredAddress(i->getBuffer()).getAddress();

         auto call = Builder.CreateCall(IGM.getEndAccessFn(), { scratch });
         call->setDoesNotThrow();
         return;
      }
   }
   llvm_unreachable("bad access enforcement");
}

void IRGenPILFunction::visitConvertFunctionInst(polar::ConvertFunctionInst *i) {
   // This instruction is specified to be a no-op.
   Explosion temp = getLoweredExplosion(i->getOperand());
   setLoweredExplosion(i, temp);
}

void IRGenPILFunction::visitConvertEscapeToNoEscapeInst(
   polar::ConvertEscapeToNoEscapeInst *i) {
   // This instruction makes the context trivial.
   Explosion in = getLoweredExplosion(i->getOperand());
   llvm::Value *fn = in.claimNext();
   llvm::Value *ctx = in.claimNext();
   Explosion out;
   out.add(fn);
   out.add(Builder.CreateBitCast(ctx, IGM.OpaquePtrTy));
   setLoweredExplosion(i, out);
}

void IRGenPILFunction::visitThinFunctionToPointerInst(
   polar::ThinFunctionToPointerInst *i) {
   Explosion in = getLoweredExplosion(i->getOperand());
   llvm::Value *fn = in.claimNext();
   fn = Builder.CreateBitCast(fn, IGM.Int8PtrTy);
   Explosion out;
   out.add(fn);
   setLoweredExplosion(i, out);
}

void IRGenPILFunction::visitPointerToThinFunctionInst(
   polar::PointerToThinFunctionInst *i) {
   Explosion in = getLoweredExplosion(i->getOperand());
   llvm::Value *fn = in.claimNext();
   fn = Builder.CreateBitCast(fn, IGM.FunctionPtrTy);
   Explosion out;
   out.add(fn);
   setLoweredExplosion(i, out);
}

void IRGenPILFunction::visitAddressToPointerInst(polar::AddressToPointerInst *i)
{
   Explosion to;
   llvm::Value *addrValue = getLoweredAddress(i->getOperand()).getAddress();
   if (addrValue->getType() != IGM.Int8PtrTy)
      addrValue = Builder.CreateBitCast(addrValue, IGM.Int8PtrTy);
   to.add(addrValue);
   setLoweredExplosion(i, to);
}

// Ignores the isStrict flag because Swift TBAA is not lowered into LLVM IR.
void IRGenPILFunction::visitPointerToAddressInst(polar::PointerToAddressInst *i)
{
   Explosion from = getLoweredExplosion(i->getOperand());
   llvm::Value *ptrValue = from.claimNext();

   auto &ti = getTypeInfo(i->getType());

   llvm::Type *destType = ti.getStorageType()->getPointerTo();
   ptrValue = Builder.CreateBitCast(ptrValue, destType);

   setLoweredAddress(i,
                     ti.getAddressForPointer(ptrValue));
}

static void emitPointerCastInst(IRGenPILFunction &IGF,
                                PILValue src,
                                PILValue dest,
                                const TypeInfo &ti) {
   Explosion from = IGF.getLoweredExplosion(src);
   llvm::Value *ptrValue = from.claimNext();
   // The input may have witness tables or other additional data, but the class
   // reference is always first.
   (void)from.claimAll();

   auto schema = ti.getSchema();
   assert(schema.size() == 1
          && schema[0].isScalar()
          && "pointer schema is not a single scalar?!");
   auto castToType = schema[0].getScalarType();

   // A retainable pointer representation may be wrapped in an optional, so we
   // need to provide inttoptr/ptrtoint in addition to bitcast.
   ptrValue = IGF.Builder.CreateBitOrPointerCast(ptrValue, castToType);

   Explosion to;
   to.add(ptrValue);
   IGF.setLoweredExplosion(dest, to);
}

void IRGenPILFunction::visitUncheckedRefCastInst(
   polar::UncheckedRefCastInst *i) {
   auto &ti = getTypeInfo(i->getType());
   emitPointerCastInst(*this, i->getOperand(), i, ti);
}

// TODO: Although runtime checks are not required, we get them anyway when
// asking the runtime to perform this cast. If this is a performance impact, we
// can add a CheckedCastMode::Unchecked.
void IRGenPILFunction::
visitUncheckedRefCastAddrInst(polar::UncheckedRefCastAddrInst *i) {
   Address dest = getLoweredAddress(i->getDest());
   Address src = getLoweredAddress(i->getSrc());
   emitCheckedCast(*this,
                   src, i->getSourceFormalType(),
                   dest, i->getTargetFormalType(),
                   CastConsumptionKind::TakeAlways,
                   CheckedCastMode::Unconditional);
}

void IRGenPILFunction::visitUncheckedAddrCastInst(
   polar::UncheckedAddrCastInst *i) {
   auto addr = getLoweredAddress(i->getOperand());
   auto &ti = getTypeInfo(i->getType());
   auto result = Builder.CreateBitCast(addr,ti.getStorageType()->getPointerTo());
   setLoweredAddress(i, result);
}

static bool isStructurallySame(const llvm::Type *T1, const llvm::Type *T2) {
   if (T1 == T2) return true;

   if (auto *S1 = dyn_cast<llvm::StructType>(T1))
      if (auto *S2 = dyn_cast<llvm::StructType>(T2))
         return S1->isLayoutIdentical(const_cast<llvm::StructType*>(S2));
   return false;
}

// Emit a trap in the event a type does not match expected layout constraints.
//
// We can hit this case in specialized functions even for correct user code.
// If the user dynamically checks for correct type sizes in the generic
// function, a specialized function can contain the (not executed) bitcast
// with mismatching fixed sizes.
// Usually llvm can eliminate this code again because the user's safety
// check should be constant foldable on llvm level.
static void emitTrapAndUndefValue(IRGenPILFunction &IGF,
                                  Explosion &in,
                                  Explosion &out,
                                  const LoadableTypeInfo &outTI) {
   llvm::BasicBlock *failBB =
      llvm::BasicBlock::Create(IGF.IGM.getLLVMContext());
   IGF.Builder.CreateBr(failBB);
   IGF.FailBBs.push_back(failBB);

   IGF.Builder.emitBlock(failBB);
   IGF.emitTrap("mismatching type layouts", /*EmitUnreachable=*/true);

   llvm::BasicBlock *contBB = llvm::BasicBlock::Create(IGF.IGM.getLLVMContext());
   IGF.Builder.emitBlock(contBB);
   (void)in.claimAll();
   for (auto schema : outTI.getSchema())
      out.add(llvm::UndefValue::get(schema.getScalarType()));
}

static void emitUncheckedValueBitCast(IRGenPILFunction &IGF,
                                      SourceLoc loc,
                                      Explosion &in,
                                      const LoadableTypeInfo &inTI,
                                      Explosion &out,
                                      const LoadableTypeInfo &outTI) {
   // If the transfer is doable bitwise, and if the elements of the explosion are
   // the same type, then just transfer the elements.
   if (inTI.isBitwiseTakable(ResilienceExpansion::Maximal) &&
       outTI.isBitwiseTakable(ResilienceExpansion::Maximal) &&
       isStructurallySame(inTI.getStorageType(), outTI.getStorageType())) {
      in.transferInto(out, in.size());
      return;
   }

   // TODO: We could do bitcasts entirely in the value domain in some cases, but
   // for simplicity, let's just always go through the stack for now.

   // Create the allocation.
   auto inStorage = IGF.createAlloca(inTI.getStorageType(),
                                     std::max(inTI.getFixedAlignment(),
                                              outTI.getFixedAlignment()),
                                     "bitcast");

   auto maxSize = std::max(inTI.getFixedSize(), outTI.getFixedSize());
   IGF.Builder.CreateLifetimeStart(inStorage, maxSize);

   // Store the 'in' value.
   inTI.initialize(IGF, in, inStorage, false);
   // Load the 'out' value as the destination type.
   auto outStorage = IGF.Builder.CreateBitCast(inStorage,
                                               outTI.getStorageType()->getPointerTo());
   outTI.loadAsTake(IGF, outStorage, out);

   IGF.Builder.CreateLifetimeEnd(inStorage, maxSize);
   return;
}

static void emitValueBitwiseCast(IRGenPILFunction &IGF,
                                 SourceLoc loc,
                                 Explosion &in,
                                 const LoadableTypeInfo &inTI,
                                 Explosion &out,
                                 const LoadableTypeInfo &outTI) {
   // Unfortunately, we can't check this invariant until we get to IRGen, since
   // the AST and PIL don't know anything about type layout.
   if (inTI.getFixedSize() < outTI.getFixedSize()) {
      emitTrapAndUndefValue(IGF, in, out, outTI);
      return;
   }
   emitUncheckedValueBitCast(IGF, loc, in, inTI, out, outTI);
}

void IRGenPILFunction::visitUncheckedTrivialBitCastInst(
   polar::UncheckedTrivialBitCastInst *i) {
   Explosion in = getLoweredExplosion(i->getOperand());
   Explosion out;

   emitValueBitwiseCast(*this, i->getLoc().getSourceLoc(),
                        in,  cast<LoadableTypeInfo>(getTypeInfo(i->getOperand()->getType())),
                        out, cast<LoadableTypeInfo>(getTypeInfo(i->getType())));

   setLoweredExplosion(i, out);
}

void IRGenPILFunction::
visitUncheckedBitwiseCastInst(polar::UncheckedBitwiseCastInst *i) {
   Explosion in = getLoweredExplosion(i->getOperand());
   Explosion out;

   emitValueBitwiseCast(*this, i->getLoc().getSourceLoc(),
                        in,  cast<LoadableTypeInfo>(getTypeInfo(i->getOperand()->getType())),
                        out, cast<LoadableTypeInfo>(getTypeInfo(i->getType())));

   setLoweredExplosion(i, out);
}

void IRGenPILFunction::visitRefToRawPointerInst(
   polar::RefToRawPointerInst *i) {
   auto &ti = getTypeInfo(i->getType());
   emitPointerCastInst(*this, i->getOperand(), i, ti);
}

void IRGenPILFunction::visitRawPointerToRefInst(polar::RawPointerToRefInst *i) {
   auto &ti = getTypeInfo(i->getType());
   emitPointerCastInst(*this, i->getOperand(), i, ti);
}

// PIL scalar conversions which never change the IR type.
// FIXME: Except for optionals, which get bit-packed into an integer.
static void trivialRefConversion(IRGenPILFunction &IGF,
                                 PILValue input,
                                 PILValue result) {
   Explosion temp = IGF.getLoweredExplosion(input);
   auto &inputTI = IGF.getTypeInfo(input->getType());
   auto &resultTI = IGF.getTypeInfo(result->getType());

   // If the types are the same, forward the existing value.
   if (inputTI.getStorageType() == resultTI.getStorageType()) {
      IGF.setLoweredExplosion(result, temp);
      return;
   }

   auto schema = resultTI.getSchema();
   Explosion out;

   for (auto schemaElt : schema) {
      auto resultTy = schemaElt.getScalarType();

      llvm::Value *value = temp.claimNext();
      if (value->getType() == resultTy) {
         // Nothing to do.  This happens with the unowned conversions.
      } else if (resultTy->isPointerTy()) {
         value = IGF.Builder.CreateIntToPtr(value, resultTy);
      } else {
         value = IGF.Builder.CreatePtrToInt(value, resultTy);
      }
      out.add(value);
   }

   IGF.setLoweredExplosion(result, out);
}

// PIL scalar conversions which never change the IR type.
// FIXME: Except for optionals, which get bit-packed into an integer.
#define NOOP_CONVERSION(KIND)                                     \
void IRGenPILFunction::visit##KIND##Inst(polar::KIND##Inst *i) {  \
  ::trivialRefConversion(*this, i->getOperand(), i); \
}
#define LOADABLE_REF_STORAGE(Name, ...) \
  NOOP_CONVERSION(Name##ToRef) \
  NOOP_CONVERSION(RefTo##Name)
#include "polarphp/ast/ReferenceStorageDef.h"
#undef NOOP_CONVERSION

void IRGenPILFunction::visitThinToThickFunctionInst(
   polar::ThinToThickFunctionInst *i) {
   // Take the incoming function pointer and add a null context pointer to it.
   Explosion from = getLoweredExplosion(i->getOperand());
   Explosion to;
   to.add(from.claimNext());
   if (i->getType().castTo<PILFunctionType>()->isNoEscape())
      to.add(llvm::ConstantPointerNull::get(IGM.OpaquePtrTy));
   else
      to.add(IGM.RefCountedNull);
   setLoweredExplosion(i, to);
}
//
//void IRGenPILFunction::visitThickToObjCMetatypeInst(ThickToObjCMetatypeInst *i){
//   Explosion from = getLoweredExplosion(i->getOperand());
//   llvm::Value *swiftMeta = from.claimNext();
//   // Claim any conformances.
//   (void)from.claimAll();
//   CanType instanceType(i->getType().castTo<AnyMetatypeType>().getInstanceType());
//   Explosion to;
//   llvm::Value *classPtr =
//      emitClassHeapMetadataRefForMetatype(*this, swiftMeta, instanceType);
//   to.add(Builder.CreateBitCast(classPtr, IGM.ObjCClassPtrTy));
//   setLoweredExplosion(i, to);
//}
// @todo
//void IRGenPILFunction::visitObjCToThickMetatypeInst(
//   ObjCToThickMetatypeInst *i) {
//   Explosion from = getLoweredExplosion(i->getOperand());
//   llvm::Value *classPtr = from.claimNext();
//
//   // Fetch the metadata for that class.
//   Explosion to;
//   auto metadata = emitObjCMetadataRefForMetadata(*this, classPtr);
//   to.add(metadata);
//   setLoweredExplosion(i, to);
//}

void IRGenPILFunction::visitUnconditionalCheckedCastInst(
   polar::UnconditionalCheckedCastInst *i) {
   Explosion value = getLoweredExplosion(i->getOperand());
   Explosion ex;
   emitScalarCheckedCast(*this, value,
                         i->getSourceLoweredType(),
                         i->getSourceFormalType(),
                         i->getTargetLoweredType(),
                         i->getTargetFormalType(),
                         CheckedCastMode::Unconditional, ex);
   setLoweredExplosion(i, ex);
}

// @todo
//
//void IRGenPILFunction::visitObjCMetatypeToObjectInst(
//   ObjCMetatypeToObjectInst *i){
//   // Bitcast the @objc metatype reference, which is already an ObjC object, to
//   // the destination type.
//   Explosion from = getLoweredExplosion(i->getOperand());
//   llvm::Value *value = from.claimNext();
//   value = Builder.CreateBitCast(value, IGM.UnknownRefCountedPtrTy);
//   Explosion to;
//   to.add(value);
//   setLoweredExplosion(i, to);
//}
//
//void IRGenPILFunction::visitObjCExistentialMetatypeToObjectInst(
//   ObjCExistentialMetatypeToObjectInst *i){
//   // Bitcast the @objc metatype reference, which is already an ObjC object, to
//   // the destination type. The metatype may carry additional witness tables we
//   // can drop.
//   Explosion from = getLoweredExplosion(i->getOperand());
//   llvm::Value *value = from.claimNext();
//   (void)from.claimAll();
//   value = Builder.CreateBitCast(value, IGM.UnknownRefCountedPtrTy);
//   Explosion to;
//   to.add(value);
//   setLoweredExplosion(i, to);
//}
//
//void IRGenPILFunction::visitObjCInterfaceInst(ObjCInterfaceInst *i) {
//   // Get the protocol reference.
//   llvm::Value *protoRef = emitReferenceToObjCInterface(*this, i->getInterface());
//   // Bitcast it to the class reference type.
//   protoRef = Builder.CreateBitCast(protoRef,
//                                    getTypeInfo(i->getType()).getStorageType());
//   Explosion ex;
//   ex.add(protoRef);
//   setLoweredExplosion(i, ex);
//}

void IRGenPILFunction::visitRefToBridgeObjectInst(
   polar::RefToBridgeObjectInst *i) {
   Explosion refEx = getLoweredExplosion(i->getConverted());
   llvm::Value *ref = refEx.claimNext();

   Explosion bitsEx = getLoweredExplosion(i->getBitsOperand());
   llvm::Value *bits = bitsEx.claimNext();

   // Mask the bits into the pointer representation.
   llvm::Value *val = Builder.CreatePtrToInt(ref, IGM.SizeTy);
   val = Builder.CreateOr(val, bits);
   val = Builder.CreateIntToPtr(val, IGM.BridgeObjectPtrTy);

   Explosion resultEx;
   resultEx.add(val);

   setLoweredExplosion(i, resultEx);
}

void IRGenPILFunction::
visitClassifyBridgeObjectInst(ClassifyBridgeObjectInst *i) {
   Explosion boEx = getLoweredExplosion(i->getOperand());
   llvm::Value *bridgeVal = boEx.claimNext();
   bridgeVal = Builder.CreatePtrToInt(bridgeVal, IGM.SizeTy);

   // This returns two bits, the first of which is "is Objective-C object", the
   // second is "is Objective-C Tagged Pointer".  Each of these bits is computed
   // by checking to see if some other bits are non-zero in the BridgeObject.
   auto bitsNonZero = [&](const SpareBitVector &bits) -> llvm::Value* {
      // If this target doesn't have the specified field, just produce false.
      if (!bits.any())
         return Builder.getInt1(0);

      llvm::Value *bitsValue =
         Builder.CreateAnd(bridgeVal, Builder.getInt(bits.asAPInt()));
      return
         Builder.CreateICmpNE(bitsValue, llvm::ConstantInt::get(IGM.SizeTy, 0));
   };

   Explosion wordEx;
   wordEx.add(bitsNonZero(IGM.TargetInfo.IsObjCPointerBit));
   wordEx.add(bitsNonZero(IGM.TargetInfo.ObjCPointerReservedBits));
   setLoweredExplosion(i, wordEx);
}

void IRGenPILFunction::visitValueToBridgeObjectInst(
   ValueToBridgeObjectInst *i) {
   Explosion in = getLoweredExplosion(i->getOperand());
   Explosion out;

   emitValueBitwiseCast(
      *this, i->getLoc().getSourceLoc(), in,
      cast<LoadableTypeInfo>(getTypeInfo(i->getOperand()->getType())), out,
      cast<LoadableTypeInfo>(getTypeInfo(i->getType())));

   setLoweredExplosion(i, out);
}

void IRGenPILFunction::visitBridgeObjectToRefInst(
   polar::BridgeObjectToRefInst *i) {
   Explosion boEx = getLoweredExplosion(i->getConverted());
   llvm::Value *bo = boEx.claimNext();
   Explosion resultEx;

   auto &refTI = getTypeInfo(i->getType());
   llvm::Type *refType = refTI.getSchema()[0].getScalarType();

   // If the value is an ObjC tagged pointer, pass it through verbatim.
   llvm::BasicBlock *taggedCont = nullptr,
      *tagged = nullptr,
      *notTagged = nullptr;
   llvm::Value *taggedRef = nullptr;
   llvm::Value *boBits = nullptr;

   ClassDecl *Cl = i->getType().getClassOrBoundGenericClass();
   if (IGM.TargetInfo.hasObjCTaggedPointers() &&
       (!Cl || !isKnownNotTaggedPointer(IGM, Cl))) {
      boBits = Builder.CreatePtrToInt(bo, IGM.SizeTy);
      APInt maskValue = IGM.TargetInfo.ObjCPointerReservedBits.asAPInt();
      llvm::Value *mask = Builder.getInt(maskValue);
      llvm::Value *reserved = Builder.CreateAnd(boBits, mask);
      llvm::Value *cond = Builder.CreateICmpEQ(reserved,
                                               llvm::ConstantInt::get(IGM.SizeTy, 0));
      tagged = createBasicBlock("tagged-pointer"),
         notTagged = createBasicBlock("not-tagged-pointer");
      taggedCont = createBasicBlock("tagged-cont");

      Builder.CreateCondBr(cond, notTagged, tagged);

      Builder.emitBlock(tagged);
      taggedRef = Builder.CreateBitCast(bo, refType);
      Builder.CreateBr(taggedCont);

      // If it's not a tagged pointer, mask off the spare bits.
      Builder.emitBlock(notTagged);
   }

   // Mask off the spare bits (if they exist).
   auto &spareBits = IGM.getHeapObjectSpareBits();
   llvm::Value *result;
   if (spareBits.any()) {
      APInt maskValue = ~spareBits.asAPInt();

      if (!boBits)
         boBits = Builder.CreatePtrToInt(bo, IGM.SizeTy);

      llvm::Value *mask = llvm::ConstantInt::get(IGM.getLLVMContext(), maskValue);
      llvm::Value *masked = Builder.CreateAnd(boBits, mask);
      result = Builder.CreateIntToPtr(masked, refType);
   } else {
      result = Builder.CreateBitCast(bo, refType);
   }

   if (taggedCont) {
      Builder.CreateBr(taggedCont);

      Builder.emitBlock(taggedCont);

      auto phi = Builder.CreatePHI(refType, 2);
      phi->addIncoming(taggedRef, tagged);
      phi->addIncoming(result, notTagged);

      result = phi;
   }

   resultEx.add(result);
   setLoweredExplosion(i, resultEx);
}

void IRGenPILFunction::visitBridgeObjectToWordInst(
   polar::BridgeObjectToWordInst *i) {
   Explosion boEx = getLoweredExplosion(i->getConverted());
   llvm::Value *val = boEx.claimNext();
   val = Builder.CreatePtrToInt(val, IGM.SizeTy);
   Explosion wordEx;
   wordEx.add(val);
   setLoweredExplosion(i, wordEx);
}

void IRGenPILFunction::visitUnconditionalCheckedCastAddrInst(
   polar::UnconditionalCheckedCastAddrInst *i) {
   Address dest = getLoweredAddress(i->getDest());
   Address src = getLoweredAddress(i->getSrc());
   emitCheckedCast(*this,
                   src, i->getSourceFormalType(),
                   dest, i->getTargetFormalType(),
                   CastConsumptionKind::TakeAlways,
                   CheckedCastMode::Unconditional);
}

void IRGenPILFunction::visitUnconditionalCheckedCastValueInst(
   polar::UnconditionalCheckedCastValueInst *i) {
   llvm_unreachable("unsupported instruction during IRGen");
}

void IRGenPILFunction::visitCheckedCastValueBranchInst(
   polar::CheckedCastValueBranchInst *i) {
   llvm_unreachable("unsupported instruction during IRGen");
}

void IRGenPILFunction::visitCheckedCastBranchInst(
   polar::CheckedCastBranchInst *i) {
   FailableCastResult castResult;
   Explosion ex;
   if (i->isExact()) {
      auto operand = i->getOperand();
      Explosion source = getLoweredExplosion(operand);
      castResult = emitClassIdenticalCast(*this, source.claimNext(),
                                          i->getSourceLoweredType(),
                                          i->getTargetLoweredType());
   } else {
      Explosion value = getLoweredExplosion(i->getOperand());
      emitScalarCheckedCast(*this, value,
                            i->getSourceLoweredType(),
                            i->getSourceFormalType(),
                            i->getTargetLoweredType(),
                            i->getTargetFormalType(),
                            CheckedCastMode::Conditional, ex);
      auto val = ex.claimNext();
      castResult.casted = val;
      llvm::Value *nil =
         llvm::ConstantPointerNull::get(cast<llvm::PointerType>(val->getType()));
      castResult.succeeded = Builder.CreateICmpNE(val, nil);
   }

   // Branch on the success of the cast.
   // All cast operations currently return null on failure.


   auto &successBB = getLoweredBB(i->getSuccessBB());
   llvm::Type *toTy = IGM.getTypeInfo(i->getTargetLoweredType()).getStorageType();
   if (toTy->isPointerTy())
      castResult.casted = Builder.CreateBitCast(castResult.casted, toTy);

   Builder.CreateCondBr(castResult.succeeded,
                        successBB.bb,
                        getLoweredBB(i->getFailureBB()).bb);

   // Feed the cast result into the nonnull branch.
   unsigned phiIndex = 0;
   Explosion ex2;
   ex2.add(castResult.casted);
   ex2.add(ex.claimAll());
   addIncomingExplosionToPHINodes(*this, successBB, phiIndex, ex2);
}

void IRGenPILFunction::visitCheckedCastAddrBranchInst(
   polar::CheckedCastAddrBranchInst *i) {
   Address dest = getLoweredAddress(i->getDest());
   Address src = getLoweredAddress(i->getSrc());
   llvm::Value *castSucceeded =
      emitCheckedCast(*this,
                      src, i->getSourceFormalType(),
                      dest, i->getTargetFormalType(),
                      i->getConsumptionKind(), CheckedCastMode::Conditional);
   Builder.CreateCondBr(castSucceeded,
                        getLoweredBB(i->getSuccessBB()).bb,
                        getLoweredBB(i->getFailureBB()).bb);
}

void IRGenPILFunction::visitKeyPathInst(polar::KeyPathInst *I) {
   auto pattern = IGM.getAddrOfKeyPathPattern(I->getPattern(), I->getLoc());
   // Build up the argument vector to instantiate the pattern here.
   Optional<StackAddress> dynamicArgsBuf;
   llvm::Value *args;
   if (!I->getSubstitutions().empty() || !I->getAllOperands().empty()) {
      auto sig = I->getPattern()->getGenericSignature();
      SubstitutionMap subs = I->getSubstitutions();

      SmallVector<GenericRequirement, 4> requirements;
      enumerateGenericSignatureRequirements(sig,
                                            [&](GenericRequirement reqt) { requirements.push_back(reqt); });

      llvm::Value *argsBufSize;
      llvm::Value *argsBufAlign;

      if (!I->getSubstitutions().empty()) {
         argsBufSize = llvm::ConstantInt::get(IGM.SizeTy,
                                              IGM.getPointerSize().getValue() * requirements.size());
         argsBufAlign = llvm::ConstantInt::get(IGM.SizeTy,
                                               IGM.getPointerAlignment().getMaskValue());
      } else {
         argsBufSize = llvm::ConstantInt::get(IGM.SizeTy, 0);
         argsBufAlign = llvm::ConstantInt::get(IGM.SizeTy, 0);
      }

      SmallVector<llvm::Value *, 4> operandOffsets;
      for (unsigned i : indices(I->getAllOperands())) {
         auto operand = I->getAllOperands()[i].get();
         auto &ti = getTypeInfo(operand->getType());
         auto ty = operand->getType();
         auto alignMask = ti.getAlignmentMask(*this, ty);
         if (i != 0) {
            auto notAlignMask = Builder.CreateNot(alignMask);
            argsBufSize = Builder.CreateAdd(argsBufSize, alignMask);
            argsBufSize = Builder.CreateAnd(argsBufSize, notAlignMask);
         }
         operandOffsets.push_back(argsBufSize);
         auto size = ti.getSize(*this, ty);
         argsBufSize = Builder.CreateAdd(argsBufSize, size);
         argsBufAlign = Builder.CreateOr(argsBufAlign, alignMask);
      }

      dynamicArgsBuf = emitDynamicAlloca(IGM.Int8Ty, argsBufSize, Alignment(16));

      Address argsBuf = dynamicArgsBuf->getAddress();

      if (!I->getSubstitutions().empty()) {
         emitInitOfGenericRequirementsBuffer(*this, requirements, argsBuf,
                                             [&](GenericRequirement reqt) -> llvm::Value * {
                                                return emitGenericRequirementFromSubstitutions(*this, sig,
                                                                                               *IGM.getTypePHPModule(),
                                                                                               reqt, subs);
                                             });
      }

      for (unsigned i : indices(I->getAllOperands())) {
         auto operand = I->getAllOperands()[i].get();
         auto &ti = getTypeInfo(operand->getType());
         auto ptr = Builder.CreateInBoundsGEP(argsBuf.getAddress(),
                                              operandOffsets[i]);
         auto addr = ti.getAddressForPointer(
            Builder.CreateBitCast(ptr, ti.getStorageType()->getPointerTo()));
         if (operand->getType().isAddress()) {
            ti.initializeWithTake(*this, addr, getLoweredAddress(operand),
                                  operand->getType(), false);
         } else {
            Explosion operandValue = getLoweredExplosion(operand);
            cast<LoadableTypeInfo>(ti).initialize(*this, operandValue, addr, false);
         }
      }
      args = argsBuf.getAddress();
   } else {
      // No arguments necessary, so the argument ought to be ignored by any
      // callbacks in the pattern.
      assert(I->getAllOperands().empty() && "indices not implemented");
      args = llvm::UndefValue::get(IGM.Int8PtrTy);
   }
   auto patternPtr = llvm::ConstantExpr::getBitCast(pattern, IGM.Int8PtrTy);
   auto call = Builder.CreateCall(IGM.getGetKeyPathFn(), {patternPtr, args});
   call->setDoesNotThrow();

   if (dynamicArgsBuf) {
      emitDeallocateDynamicAlloca(*dynamicArgsBuf);
   }

   auto resultStorageTy = IGM.getTypeInfo(I->getType()).getStorageType();

   Explosion e;
   e.add(Builder.CreateBitCast(call, resultStorageTy));
   setLoweredExplosion(I, e);
}

void IRGenPILFunction::visitUpcastInst(polar::UpcastInst *i) {
   auto toTy = getTypeInfo(i->getType()).getSchema()[0].getScalarType();

   // If we have an address, just bitcast, don't explode.
   if (i->getOperand()->getType().isAddress()) {
      Address fromAddr = getLoweredAddress(i->getOperand());
      llvm::Value *toValue = Builder.CreateBitCast(
         fromAddr.getAddress(), toTy->getPointerTo());
      Address Addr(toValue, fromAddr.getAlignment());
      setLoweredAddress(i, Addr);
      return;
   }

   Explosion from = getLoweredExplosion(i->getOperand());
   Explosion to;
   assert(from.size() == 1 && "class should explode to single value");
   llvm::Value *fromValue = from.claimNext();
   to.add(Builder.CreateBitCast(fromValue, toTy));
   setLoweredExplosion(i, to);
}

void IRGenPILFunction::visitIndexAddrInst(polar::IndexAddrInst *i) {
   Address base = getLoweredAddress(i->getBase());
   Explosion indexValues = getLoweredExplosion(i->getIndex());
   llvm::Value *index = indexValues.claimNext();

   auto baseTy = i->getBase()->getType();
   auto &ti = getTypeInfo(baseTy);

   Address dest = ti.indexArray(*this, base, index, baseTy);
   setLoweredAddress(i, dest);
}

void IRGenPILFunction::visitTailAddrInst(polar::TailAddrInst *i) {
   Address base = getLoweredAddress(i->getBase());
   Explosion indexValues = getLoweredExplosion(i->getIndex());
   llvm::Value *index = indexValues.claimNext();

   PILType baseTy = i->getBase()->getType();
   const TypeInfo &baseTI = getTypeInfo(baseTy);

   Address dest = baseTI.indexArray(*this, base, index, baseTy);
   const TypeInfo &TailTI = getTypeInfo(i->getTailType());
   dest = TailTI.roundUpToTypeAlignment(*this, dest, i->getTailType());
   llvm::Type *destType = TailTI.getStorageType()->getPointerTo();
   dest = Builder.CreateBitCast(dest, destType);
   setLoweredAddress(i, dest);
}

void IRGenPILFunction::visitIndexRawPointerInst(polar::IndexRawPointerInst *i) {
   Explosion baseValues = getLoweredExplosion(i->getBase());
   llvm::Value *base = baseValues.claimNext();

   Explosion indexValues = getLoweredExplosion(i->getIndex());
   llvm::Value *index = indexValues.claimNext();

   // We don't expose a non-inbounds GEP operation.
   llvm::Value *destValue = Builder.CreateInBoundsGEP(base, index);

   Explosion result;
   result.add(destValue);
   setLoweredExplosion(i, result);
}

void IRGenPILFunction::visitAllocValueBufferInst(
   polar::AllocValueBufferInst *i) {
   Address buffer = getLoweredAddress(i->getOperand());
   auto valueType = i->getValueType();
   Address value = emitAllocateValueInBuffer(*this, valueType, buffer);
   setLoweredAddress(i, value);
}

void IRGenPILFunction::visitProjectValueBufferInst(
   polar::ProjectValueBufferInst *i) {
   Address buffer = getLoweredAddress(i->getOperand());
   auto valueType = i->getValueType();
   Address value = emitProjectValueInBuffer(*this, valueType, buffer);
   setLoweredAddress(i, value);
}

void IRGenPILFunction::visitDeallocValueBufferInst(
   polar::DeallocValueBufferInst *i) {
   Address buffer = getLoweredAddress(i->getOperand());
   auto valueType = i->getValueType();
   emitDeallocateValueInBuffer(*this, valueType, buffer);
}

void IRGenPILFunction::visitInitExistentialAddrInst(polar::InitExistentialAddrInst *i) {
   Address container = getLoweredAddress(i->getOperand());
   PILType destType = i->getOperand()->getType();
   emitOpaqueExistentialContainerInit(
      *this, container, destType, i->getFormalConcreteType(),
      i->getLoweredConcreteType(), i->getConformances());
   auto srcType = i->getLoweredConcreteType();

   // Allocate a COW box for the value if necessary.
   auto *genericEnv = CurPILFn->getGenericEnvironment();
   setLoweredAddress(
      i, emitAllocateBoxedOpaqueExistentialBuffer(
         *this, destType, srcType, container, genericEnv, false));
}

void IRGenPILFunction::visitInitExistentialValueInst(
   polar::InitExistentialValueInst *i) {
   llvm_unreachable("unsupported instruction during IRGen");
}

void IRGenPILFunction::visitInitExistentialMetatypeInst(
   InitExistentialMetatypeInst *i) {
   Explosion metatype = getLoweredExplosion(i->getOperand());
   Explosion result;
   emitExistentialMetatypeContainer(*this,
                                    result, i->getType(),
                                    metatype.claimNext(),
                                    i->getOperand()->getType(),
                                    i->getConformances());
   setLoweredExplosion(i, result);
}

void IRGenPILFunction::visitInitExistentialRefInst(InitExistentialRefInst *i) {
   Explosion instance = getLoweredExplosion(i->getOperand());
   Explosion result;
   emitClassExistentialContainer(*this,
                                 result, i->getType(),
                                 instance.claimNext(),
                                 i->getFormalConcreteType(),
                                 i->getOperand()->getType(),
                                 i->getConformances());
   setLoweredExplosion(i, result);
}

void IRGenPILFunction::visitDeinitExistentialAddrInst(
   polar::DeinitExistentialAddrInst *i) {
   Address container = getLoweredAddress(i->getOperand());

   // Deallocate the COW box for the value if necessary.
   emitDeallocateBoxedOpaqueExistentialBuffer(*this, i->getOperand()->getType(),
                                              container);
}

void IRGenPILFunction::visitDeinitExistentialValueInst(
   polar::DeinitExistentialValueInst *i) {
   llvm_unreachable("unsupported instruction during IRGen");
}

void IRGenPILFunction::visitOpenExistentialAddrInst(OpenExistentialAddrInst *i) {
   PILType baseTy = i->getOperand()->getType();
   Address base = getLoweredAddress(i->getOperand());

   auto openedArchetype = i->getType().castTo<ArchetypeType>();

   // Insert a copy of the boxed value for COW semantics if necessary.
   auto accessKind = i->getAccessKind();
   Address object = emitOpaqueBoxedExistentialProjection(
      *this, accessKind, base, baseTy, openedArchetype);

   setLoweredAddress(i, object);
}

void IRGenPILFunction::visitOpenExistentialRefInst(OpenExistentialRefInst *i) {

   PILType baseTy = i->getOperand()->getType();
   Explosion base = getLoweredExplosion(i->getOperand());
   auto openedArchetype = i->getType().castTo<ArchetypeType>();

   Explosion result;
   llvm::Value *instance
      = emitClassExistentialProjection(*this, base, baseTy,
                                       openedArchetype);
   result.add(instance);
   setLoweredExplosion(i, result);
}

void IRGenPILFunction::visitOpenExistentialMetatypeInst(
   OpenExistentialMetatypeInst *i) {
   PILType baseTy = i->getOperand()->getType();
   Explosion base = getLoweredExplosion(i->getOperand());
   auto openedTy = i->getType().getAstType();

   llvm::Value *metatype =
      emitExistentialMetatypeProjection(*this, base, baseTy, openedTy);
   Explosion result;
   result.add(metatype);
   setLoweredExplosion(i, result);
}

void IRGenPILFunction::visitOpenExistentialValueInst(
   OpenExistentialValueInst *i) {
   llvm_unreachable("unsupported instruction during IRGen");
}

void IRGenPILFunction::visitProjectBlockStorageInst(ProjectBlockStorageInst *i){
   // TODO
   Address block = getLoweredAddress(i->getOperand());
   Address capture = projectBlockStorageCapture(*this, block,
                                                i->getOperand()->getType().castTo<PILBlockStorageType>());

   setLoweredAddress(i, capture);
}

void IRGenPILFunction::visitInitBlockStorageHeaderInst(
   InitBlockStorageHeaderInst *i) {
   auto addr = getLoweredAddress(i->getBlockStorage());

   // We currently only support static invoke functions.
   auto &invokeVal = getLoweredValue(i->getInvokeFunction());
   llvm::Constant *invokeFn = nullptr;
   ForeignFunctionInfo foreignInfo;
   if (invokeVal.kind != LoweredValue::Kind::FunctionPointer) {
      IGM.unimplemented(i->getLoc().getSourceLoc(),
                        "non-static block invoke function");
   } else {
      auto &fn = invokeVal.getFunctionPointer();
      invokeFn = fn.getDirectPointer();
      foreignInfo = fn.getForeignInfo();
   }

   assert(foreignInfo.ClangInfo && "no clang info for block function?");

   // Initialize the header.
   emitBlockHeader(*this, addr,
                   i->getBlockStorage()->getType().castTo<PILBlockStorageType>(),
                   invokeFn, i->getInvokeFunction()->getType().castTo<PILFunctionType>(),
                   foreignInfo);

   // Cast the storage to the block type to produce the result value.
   llvm::Value *asBlock = Builder.CreateBitCast(addr.getAddress(),
                                                IGM.ObjCBlockPtrTy);
   Explosion e;
   e.add(asBlock);
   setLoweredExplosion(i, e);
}

void IRGenPILFunction::visitAllocExistentialBoxInst(AllocExistentialBoxInst *i){
   OwnedAddress boxWithAddr =
      emitBoxedExistentialContainerAllocation(*this, i->getExistentialType(),
                                              i->getFormalConcreteType(),
                                              i->getConformances());
   setLoweredBox(i, boxWithAddr);
}

void IRGenPILFunction::visitDeallocExistentialBoxInst(
   DeallocExistentialBoxInst *i) {
   Explosion box = getLoweredExplosion(i->getOperand());
   emitBoxedExistentialContainerDeallocation(*this, box,
                                             i->getOperand()->getType(),
                                             i->getConcreteType());
}

void IRGenPILFunction::visitOpenExistentialBoxInst(OpenExistentialBoxInst *i) {
   Explosion box = getLoweredExplosion(i->getOperand());
   auto openedArchetype = i->getType().castTo<ArchetypeType>();

   auto addr = emitOpenExistentialBox(*this, box, i->getOperand()->getType(),
                                      openedArchetype);
   setLoweredAddress(i, addr);
}

void IRGenPILFunction::visitOpenExistentialBoxValueInst(
   OpenExistentialBoxValueInst *i) {
   llvm_unreachable("unsupported instruction during IRGen");
}

void
IRGenPILFunction::visitProjectExistentialBoxInst(ProjectExistentialBoxInst *i) {
   const LoweredValue &val = getLoweredValue(i->getOperand());
   if (val.isBoxWithAddress()) {
      // The operand is an alloc_existential_box.
      // We can directly reuse the address.
      setLoweredAddress(i, val.getAddressOfBox());
   } else {
      Explosion box = getLoweredExplosion(i->getOperand());
      auto caddr = emitBoxedExistentialProjection(*this, box,
                                                  i->getOperand()->getType(),
                                                  i->getType().getAstType());
      setLoweredAddress(i, caddr.getAddress());
   }
}

void IRGenPILFunction::visitWitnessMethodInst(polar::WitnessMethodInst *i) {
   CanType baseTy = i->getLookupType();
   InterfaceConformanceRef conformance = i->getConformance();
   PILDeclRef member = i->getMember();

   assert(member.requiresNewWitnessTableEntry());

   if (IGM.isResilient(conformance.getRequirement(),
                       ResilienceExpansion::Maximal)) {
      auto *fnPtr = IGM.getAddrOfDispatchThunk(member, NotForDefinition);
      auto fnType = IGM.getPILTypes().getConstantFunctionType(
         IGM.getMaximalTypeExpansionContext(), member);
      auto sig = IGM.getSignature(fnType);
      auto fn = FunctionPointer::forDirect(fnPtr, sig);

      setLoweredFunctionPointer(i, fn);
      return;
   }

   // It would be nice if this weren't discarded.
   llvm::Value *baseMetadataCache = nullptr;

   auto fn = emitWitnessMethodValue(*this, baseTy, &baseMetadataCache, member,
                                    conformance);

   setLoweredFunctionPointer(i, fn);
}

void IRGenPILFunction::setAllocatedAddressForBuffer(PILValue v,
                                                    const Address &allocedAddress) {
   overwriteAllocatedAddress(v, allocedAddress);

   // Emit the debug info for the variable if any.
   if (auto allocStack = dyn_cast<AllocStackInst>(v)) {
      emitDebugInfoForAllocStack(allocStack, getTypeInfo(v->getType()),
                                 allocedAddress.getAddress());
   }
}

void IRGenPILFunction::visitCopyAddrInst(polar::CopyAddrInst *i) {
   PILType addrTy = i->getSrc()->getType();
   const TypeInfo &addrTI = getTypeInfo(addrTy);
   Address src = getLoweredAddress(i->getSrc());
   // See whether we have a deferred fixed-size buffer initialization.
   auto &loweredDest = getLoweredValue(i->getDest());
   assert(!loweredDest.isUnallocatedAddressInBuffer());
   Address dest = loweredDest.getAnyAddress();
   if (i->isInitializationOfDest()) {
      if (i->isTakeOfSrc()) {
         addrTI.initializeWithTake(*this, dest, src, addrTy, false);
      } else {
         addrTI.initializeWithCopy(*this, dest, src, addrTy, false);
      }
   } else {
      if (i->isTakeOfSrc()) {
         addrTI.assignWithTake(*this, dest, src, addrTy, false);
      } else {
         addrTI.assignWithCopy(*this, dest, src, addrTy, false);
      }
   }
}

// This is a no-op because we do not lower Swift TBAA info to LLVM IR, and it
// does not produce any values.
void IRGenPILFunction::visitBindMemoryInst(polar::BindMemoryInst *) {}

void IRGenPILFunction::visitDestroyAddrInst(polar::DestroyAddrInst *i) {
   PILType addrTy = i->getOperand()->getType();
   const TypeInfo &addrTI = getTypeInfo(addrTy);

   Address base = getLoweredAddress(i->getOperand());
   addrTI.destroy(*this, base, addrTy, false /*isOutlined*/);
}

void IRGenPILFunction::visitCondFailInst(polar::CondFailInst *i) {
   Explosion e = getLoweredExplosion(i->getOperand());
   llvm::Value *cond = e.claimNext();

   // The condition should be false, or we die.
   auto expectedCond = Builder.CreateExpect(cond,
                                            llvm::ConstantInt::get(IGM.Int1Ty, 0));

   // Emit individual fail blocks so that we can map the failure back to a source
   // line.
   auto origInsertionPoint = Builder.GetInsertBlock();

   llvm::BasicBlock *failBB = llvm::BasicBlock::Create(IGM.getLLVMContext());
   llvm::BasicBlock *contBB = llvm::BasicBlock::Create(IGM.getLLVMContext());
   Builder.CreateCondBr(expectedCond, failBB, contBB);

   Builder.SetInsertPoint(&CurFn->back());
   Builder.emitBlock(failBB);
   if (IGM.DebugInfo)
      // If we are emitting DWARF, this does nothing. Otherwise the ``llvm.trap``
      // instruction emitted from ``Builtin.condfail`` should have an inlined
      // debug location. This is because zero is not an artificial line location
      // in CodeView.
      IGM.DebugInfo->setInlinedTrapLocation(Builder, i->getDebugScope());
   emitTrap(i->getMessage(), /*EmitUnreachable=*/true);

   Builder.SetInsertPoint(origInsertionPoint);
   Builder.emitBlock(contBB);
   FailBBs.push_back(failBB);
}

void IRGenPILFunction::visitSuperMethodInst(polar::SuperMethodInst *i) {
   assert(!i->getMember().isForeign);

   auto base = getLoweredExplosion(i->getOperand());
   auto baseType = i->getOperand()->getType();
   llvm::Value *baseValue = base.claimNext();

   auto method = i->getMember().getOverriddenVTableEntry();
   auto methodType = i->getType().castTo<PILFunctionType>();

   auto *classDecl = cast<ClassDecl>(method.getDecl()->getDeclContext());

   // If the class defining the vtable entry is resilient, we cannot assume
   // its offset since methods can be re-ordered resiliently. Instead, we call
   // the class method lookup function, passing in a reference to the
   // method descriptor.
   if (IGM.hasResilientMetadata(classDecl, ResilienceExpansion::Maximal)) {
      // Load the superclass of the static type of the 'self' value.
      llvm::Value *superMetadata;
      auto instanceTy = CanType(baseType.getAstType()->getMetatypeInstanceType());
      if (!IGM.hasResilientMetadata(instanceTy.getClassOrBoundGenericClass(),
                                    ResilienceExpansion::Maximal)) {
         // It's still possible that the static type of 'self' is not resilient, in
         // which case we can assume its superclass.
         //
         // An example is the following hierarchy, where ModuleA is resilient and
         // we're inside ModuleB:
         //
         // ModuleA.Base <-- defines method
         // |
         // \- ModuleB.Middle
         //    |
         //    \- ModuleB.Derived <-- static type of 'self'
         //
         // It's OK to know that the superclass of Derived is Middle, but the
         // method requires using a resilient access pattern.
         auto superTy = instanceTy->getSuperclass();
         superMetadata = emitClassHeapMetadataRef(*this, superTy->getCanonicalType(),
                                                  MetadataValueType::TypeMetadata,
                                                  MetadataState::Complete);
      } else {
         // Otherwise, we're in the most general case; the superclass might change,
         // so we have to load it dynamically from the metadata of the static type
         // of 'self'.
         auto *metadata = emitClassHeapMetadataRef(*this, instanceTy,
                                                   MetadataValueType::TypeMetadata,
                                                   MetadataState::Complete);

         auto superField = emitAddressOfSuperclassRefInClassMetadata(*this, metadata);
         superMetadata = Builder.CreateLoad(superField);
      }

      // Get the method descriptor.
      auto *methodDescriptor =
         IGM.getAddrOfMethodDescriptor(method, NotForDefinition);

      // Get the method lookup function for the class defining the method.
      auto *lookupFn = IGM.getAddrOfMethodLookupFunction(classDecl,
                                                         NotForDefinition);

      // Call the lookup function.
      llvm::Value *fnPtr = Builder.CreateCall(lookupFn,
                                              {superMetadata, methodDescriptor});

      // The function returns an i8*; cast it to the correct type.
      auto sig = IGM.getSignature(methodType);
      fnPtr = Builder.CreateBitCast(fnPtr, sig.getType()->getPointerTo());

      FunctionPointer fn(fnPtr, sig);

      setLoweredFunctionPointer(i, fn);
      return;
   }

   // Non-resilient case.

   auto fn =
      emitVirtualMethodValue(*this, baseValue, baseType, method, methodType,
         /*useSuperVTable*/ true);

   setLoweredFunctionPointer(i, fn);
}

void IRGenPILFunction::visitObjCSuperMethodInst(polar::ObjCSuperMethodInst *i) {
   assert(i->getMember().isForeign);
   // @todo
//   setLoweredObjCMethodBounded(i, i->getMember(),
//                               i->getOperand()->getType(),
//      /*startAtSuper=*/true);
}

void IRGenPILFunction::visitClassMethodInst(polar::ClassMethodInst *i) {
   assert(!i->getMember().isForeign);

   Explosion base = getLoweredExplosion(i->getOperand());
   llvm::Value *baseValue = base.claimNext();

   PILDeclRef method = i->getMember().getOverriddenVTableEntry();
   auto methodType = i->getType().castTo<PILFunctionType>();

   auto *classDecl = cast<ClassDecl>(method.getDecl()->getDeclContext());
   if (IGM.hasResilientMetadata(classDecl,
                                ResilienceExpansion::Maximal)) {
      auto *fnPtr = IGM.getAddrOfDispatchThunk(method, NotForDefinition);
      auto sig = IGM.getSignature(methodType);
      FunctionPointer fn(fnPtr, sig);

      setLoweredFunctionPointer(i, fn);
      return;
   }

   // For Swift classes, get the method implementation from the vtable.
   // FIXME: better explosion kind, map as static.
   FunctionPointer fn = emitVirtualMethodValue(
      *this, baseValue, i->getOperand()->getType(), method, methodType,
      /*useSuperVTable*/ false);

   setLoweredFunctionPointer(i, fn);
}

void IRGenPILFunction::visitObjCMethodInst(polar::ObjCMethodInst *i) {
   // For Objective-C classes we need to arrange for a msgSend
   // to happen when the method is called.
   assert(i->getMember().isForeign);
//   setLoweredObjCMethod(i, i->getMember());
}

void IRGenModule::emitPILStaticInitializers() {
   SmallVector<PILFunction *, 8> StaticInitializers;
   for (PILGlobalVariable &Global : getPILModule().getPILGlobals()) {
      PILInstruction *InitValue = Global.getStaticInitializerValue();
      if (!InitValue)
         continue;

#ifndef NDEBUG
      PILType loweredTy = Global.getLoweredType();
      auto &ti = getTypeInfo(loweredTy);

      auto expansion = getResilienceExpansionForLayout(&Global);
      assert(ti.isFixedSize(expansion) &&
             "cannot emit a static initializer for dynamically-sized global");
#endif

      auto *IRGlobal =
         Module.getGlobalVariable(Global.getName(), true /* = AllowLocal */);

      // A check for multi-threaded compilation: Is this the llvm module where the
      // global is defined and not only referenced (or not referenced at all).
      if (!IRGlobal || !IRGlobal->hasInitializer())
         continue;

      if (auto *OI = dyn_cast<ObjectInst>(InitValue)) {
         StructLayout *Layout = StaticObjectLayouts[&Global].get();
         llvm::Constant *InitVal = emitConstantObject(*this, OI, Layout);
         auto *ContainerTy = cast<llvm::StructType>(IRGlobal->getValueType());
         auto *zero = llvm::ConstantAggregateZero::get(ContainerTy->getElementType(0));
         IRGlobal->setInitializer(llvm::ConstantStruct::get(ContainerTy,
                                                            {zero , InitVal}));
         continue;
      }

      // Set the IR global's initializer to the constant for this PIL
      // struct.
      if (auto *SI = dyn_cast<StructInst>(InitValue)) {
         IRGlobal->setInitializer(emitConstantStruct(*this, SI));
         continue;
      }

      // Set the IR global's initializer to the constant for this PIL
      // tuple.
      auto *TI = cast<TupleInst>(InitValue);
      IRGlobal->setInitializer(emitConstantTuple(*this, TI));
   }
}
